+
    TV-j 7  ã                   óv  € ^ RI Ht ^ RIHt ^ RIHt ^ RIHt ^ RI	H
t
 ^RIHt ^RIHt ] ! R R]4      4       t]! ]P                   R	R
7      R 4       t]! ]P                   R	R
7      R 4       t]P                   R 4       tR t]! 4       tR R lt ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R]
P.                  4      t ! R R ]
P.                  4      tR# )!é    )Ú	dataclass)Úpartial)ÚOptionalN)ÚBaseModelArgs)ÚArraysCachec                   ó0   a € ] tR t^t o RtV 3R ltRtV tR# )Ú	ModelArgsFc                óª   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnorm_epsÚhead_dimÚnum_hidden_layersÚa_low_rank_dimÚv_low_rank_dimÚgate_low_rank_dimÚdecay_low_rank_dimÚtie_word_embeddings)ÚstrÚintÚfloatÚbool)ÚformatÚ__classdict__s   "€Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/rwkv7.pyÚ__annotate__ÚModelArgs.__annotate__   s›   ø‡ ‚ á�Oñ ñ �Oñ ñ Ññ	 ñ
 Ññ ñ �Oñ ñ �Mñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ñ%ò ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r   s   @r   r	   r	      s   ø‡ € ð !&Ð÷ ƒ r!   r	   T)Ú	shapelessc                 ó    € WV,          ,           # ©Nr"   )ÚxÚyÚzs   &&&r   Úaddcmulr1      s   € à�1�u�9Ðr!   c           	      ó~   € V \         P                  ! \         P                  P                  V RRR7      R4      ,          # )é   T©ÚaxisÚkeepdimsgH¯¼šò×z>éÿÿÿÿ)ÚmxÚmaximumÚlinalgÚnorm)r.   s   &r   Úl2_normr<   #   s+   € àŒr�zŠzœ"Ÿ)™)Ÿ.™.¨°¸d˜.ÓCÀTÓJÕJÐJr!   c                 óÄ   € WdR,          ,          VR,          ,          pWaR,          ,          VR,          VR,          ,          ,           V,           pW`R,          ,          pW†3# ).).N).NºNNN)r>   r>   Nr>   r"   )	ÚrÚwÚkÚvÚaÚbÚstateÚsabr/   s	   &&&&&&&  r   Ú_wkv7_step_opsrG   (   sN   € à�Y•<Õ 1 \¥?Õ
2€CØ�mÕ$Õ$ q¨¥|°a¸µoÕ'EÕEÈÕK€EØ�)•Õ€AØˆ8€Or!   c                  ó¤   € \         P                  P                  4       '       g   R # Rp . ROp\         P                  P	                  RVRR.V R7      # )Na÷  
        auto n = thread_position_in_grid.z;
        auto b_idx = n / H;
        auto h_idx = n % H;
        constexpr int n_per_t = D / 32;

        // [B, T, H, D]
        auto r_ = r + b_idx * T * H * D + h_idx * D;
        auto w_ = w + b_idx * T * H * D + h_idx * D;
        auto k_ = k + b_idx * T * H * D + h_idx * D;
        auto v_ = v + b_idx * T * H * D + h_idx * D;
        auto a_ = a + b_idx * T * H * D + h_idx * D;
        auto b_ = b + b_idx * T * H * D + h_idx * D;
        y += b_idx * T * H * D + h_idx * D;

        auto dk_idx = thread_position_in_threadgroup.x;
        auto dv_idx = thread_position_in_grid.y;

        // state_in, state_out: [B, H, D, D]
        auto i_state = state_in  + (n * D + dv_idx) * D;
        auto o_state = state_out + (n * D + dv_idx) * D;

        float state[n_per_t];
        for (int i = 0; i < n_per_t; ++i) {
          auto s_idx = n_per_t * dk_idx + i;
          state[i] = static_cast<float>(i_state[s_idx]);
        }

        for (int t = 0; t < T; ++t) {
          float sa = 0.0f;
          for (int i = 0; i < n_per_t; ++i) {
            auto s_idx = n_per_t * dk_idx + i;
            sa += state[i] * a_[s_idx];
            state[i] = state[i] * w_[s_idx];
          }
          sa = simd_sum(sa);

          float out = 0.0f;
          for (int i = 0; i < n_per_t; ++i) {
            auto s_idx = n_per_t * dk_idx + i;
            state[i] = state[i] + k_[s_idx] * v_[dv_idx] + sa * b_[s_idx];
            out += state[i] * r_[s_idx];
          }
          out = simd_sum(out);
          if (thread_index_in_simdgroup == 0) {
            y[dv_idx] = static_cast<InT>(out);
          }

          // Increment data pointers to next time step
          r_ += H * D;
          w_ += H * D;
          k_ += H * D;
          v_ += H * D;
          a_ += H * D;
          b_ += H * D;
          y  += H * D;
        }
        for (int i = 0; i < n_per_t; ++i) {
          auto s_idx = n_per_t * dk_idx + i;
          o_state[s_idx] = static_cast<InT>(state[i]);
        }
    Úwkv7_kernelr/   Ú	state_out)ÚnameÚinput_namesÚoutput_namesÚsource)r?   r@   rA   rB   rC   rD   Ústate_inÚT)r8   ÚmetalÚis_availableÚfastÚmetal_kernel)rN   Úinputss     r   Ú_make_wkv7_kernelrV   0   sT   € Ü�8‰8× Ñ ×"Ò"Ùð=€Fò| =€FÜ�7‰7×ÑØØØ˜;Ð'Øð	  ó ð r!   c                óø   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R\         P                  /# )r   r?   r@   rA   rB   rC   rD   rE   ©r8   Úarray)r   s   "r   r   r   }   sj   € ÷ ñ Ü	‡x�xðä	‡x�xðô 
‡x�xðô 
‡x�xð	ô
 
‡x�xðô 
‡x�xðô �8‰8ñr!   c           
      ó¤   € V P                   w  rxršV P                  p\        WW#WEWh.R V3RV	3RV
3.^ W§V	,          3RWxWš3VP                   .W».R7      # )ÚInTÚHÚD)rU   ÚtemplateÚgridÚthreadgroupÚoutput_shapesÚoutput_dtypes)é    é   r3   )ÚshapeÚdtypeÚ_wkv7_kernel)r?   r@   rA   rB   rC   rD   rE   ÚBrP   r\   r]   Úinput_dtypes   &&&&&&&     r   rI   rI   }   st   € ð —‘�J€Aˆ!Ø—'‘'€KäØ�a˜A %Ð+à�KÐ Ø�!ˆHØ�!ˆHð
ð
 �!˜•Uˆ^ØØ˜a�| U§[¡[Ð1Ø"Ð0ôð r!   c                   ó8   a a€ ] tR t^—t oV 3R ltR tRtVtV ;t# )ÚLayerNormPerHeadc                ó    <€ \         SV `  4        \        P                  ! W!34      V n        \        P                  ! W!34      V n        W0n        R # r-   )ÚsuperÚ__init__r8   ÚzerosÚweightÚbiasÚeps)Úselfr   Ú	num_headsrr   Ú	__class__s   &&&&€r   rn   ÚLayerNormPerHead.__init__˜   s9   ø€ Ü‰ÑÔÜ—h’h 	Ð4Ó5ˆŒÜ—H’H˜iÐ2Ó3ˆŒ	ØŽr!   c                óž   € V P                   \        P                  P                  VR R V P                  4      ,          V P
                  ,           # r-   )rp   r8   rS   Ú
layer_normrr   rq   ©rs   r.   s   &&r   Ú__call__ÚLayerNormPerHead.__call__ž   s3   € Ø�{‰{œRŸW™W×/Ñ/°°4¸¸t¿x¹xÓHÕHÈ4Ï9É9ÕTÐTr!   )rq   rr   rp   ©	r#   r$   r%   r&   rn   rz   r(   r)   Ú__classcell__©ru   r   s   @@r   rk   rk   —   s   ù‡ € õ÷Uò Ur!   rk   c                   óT   a a€ ] tR t^¢t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚLoRAc                óX   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[,          RS[S[,          /# )r   Ú	input_dimÚ
output_dimÚlow_rank_dimrq   Ú
activation)r   r   r   r   )r   r   s   "€r   r   ÚLoRA.__annotate__£   sE   ø€ ÷ 
ñ 
áð
ñ ð
ñ ð	
ñ
 ‘t�nð
ñ ™S•Mñ
r!   c                ód  <€ \         SV `  4        Wn        W n        W0n        W@n        Vf   \        P                  ! 4       V n        MuVR8X  d   \        P                  ! 4       V n        MSVR8X  d   \        P                  ! 4       V n        M1VR8X  d   \        P                  ! 4       V n        M\        RV R24      h\        P                  ! V P                  V P                  RR7      V P                  \        P                  ! V P                  V P                  V P
                  R7      .V n        R # )NÚsigmoidÚtanhÚreluzUnsupported activation type: Ú.F©rq   )rm   rn   r‚   rƒ   r„   rq   ÚnnÚIdentityr…   ÚSigmoidÚTanhÚReLUÚ
ValueErrorÚLinearÚlora)rs   r‚   rƒ   r„   rq   r…   ru   s   &&&&&&€r   rn   ÚLoRA.__init__£   sÖ   ø€ ô 	‰ÑÔØ"ŒØ$ŒØ(ÔØŒ	àÒÜ Ÿkšk›mˆD�OØ˜9Ô$Ü Ÿjšj›lˆD�OØ˜6Ô!Ü Ÿgšg›iˆD�OØ˜6Ô!Ü Ÿgšg›iˆD�OäÐ<¸Z¸LÈÐJÓKÐKô �IŠI�d—n‘n d×&7Ñ&7¸eÔDØ�O‰OÜ�IŠI�d×'Ñ'¨¯©¸t¿y¹yÔIð
ˆŽ	r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   ÚreturnrX   )r   r   s   "€r   r   r†   Â   s   ø€ ÷ ;ñ ;™RŸX™Xñ ;r!   c                ó�   € V P                   ^,          ! V P                   ^,          ! V P                   ^ ,          ! V4      4      4      # )r   )r”   ry   s   &&r   rz   ÚLoRA.__call__Â   s.   € Ø�y‰y˜Ž|˜DŸI™I ažL¨¯©°1®°a«Ó9Ó:Ð:r!   )r…   rq   r‚   r”   r„   rƒ   )Tr‰   r|   r~   s   @@r   r€   r€   ¢   s   ù‡ € ÷
õ 
÷>;÷ ;ð ;r!   r€   c                   ó&   a € ] tR t^Æt o R tRtV tR# )Ú
TokenShiftc                óÌ   € VP                   w  r4pVf%   \        P                  ! V^V3VP                  4      pV^8X  d   V# \        P                  ! W!RR R1R3,          .^R7      # )Nr>   ©r5   r7   )re   r8   ro   rf   Úconcatenate)rs   r.   rE   rh   ÚLr]   s   &&&   r   rz   ÚTokenShift.__call__Ç   s\   € Ø—'‘'‰ˆˆaØŠ=Ü—H’H˜a  A˜Y¨¯©Ó0ˆEØ�Œ6ØˆLä—>’> 5¨A¨s°¨s°A¨I­,Ð"7¸aÔ@Ð@r!   r"   N)r#   r$   r%   r&   rz   r(   r)   r*   s   @r   rœ   rœ   Æ   s   ø‡ € ÷Að Ar!   rœ   c                   óP   a a€ ] tR t^Ñt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚRwkv7ChannelMixingc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r	   )r   r   s   "€r   r   ÚRwkv7ChannelMixing.__annotate__Ò   s   ø€ ÷ (ñ (™Yñ (r!   c                ó  <€ \         SV `  4        VP                  pVP                  p\        P
                  ! W#R R7      V n        \        P
                  ! W2R R7      V n        \        P                  ! V4      V n
        \        4       V n        R# ©FrŒ   N)rm   rn   r   r   r�   r“   ÚkeyÚvaluer8   ro   Úx_krœ   Útoken_shift)rs   r¦   Ú
hidden_dimr   ru   s   &&  €r   rn   ÚRwkv7ChannelMixing.__init__Ò   sd   ø€ Ü‰ÑÔà×%Ñ%ˆ
Ø ×2Ñ2Ðä—9’9˜ZÀÔGˆŒÜ—Y’YÐ0À5ÔIˆŒ
ä—8’8˜ZÓ)ˆŒä%›<ˆÖr!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r—   rX   )r   r   s   "€r   r   r¨   ß   s   ø€ ÷ 2ñ 2¡B§H¡Hñ 2r!   c                ó  € Ve
   V^,          MR pV P                  W4      p\        WV,
          V P                  4      pVe   VRRR 1R3,          V^&   V P                  \        P
                  ! V P                  V4      4      4      # )Nr>   r7   )r®   r1   r­   r¬   r�   Úrelu2r«   )rs   r.   ÚcacherE   Úx_prevÚxxs   &&&   r   rz   ÚRwkv7ChannelMixing.__call__ß   sq   € Ø!Ò-��a–°4ˆØ×!Ñ! !Ó+ˆÜ�Q �
 D§H¡HÓ-ˆØÒØ˜˜B™C ˜•|ˆE�!‰HØ�z‰zœ"Ÿ(š( 4§8¡8¨B£<Ó0Ó1Ð1r!   )r«   r®   r¬   r­   r|   r~   s   @@r   r£   r£   Ñ   s   ù‡ € ÷(ó (÷2÷ 2ð 2r!   r£   c                   óJ   a a€ ] tR t^èt oV3R lV 3R lltR tR tRtVtV ;t	# )ÚRwkv7TimeMixingc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   r¦   Ú	layer_idx©r	   r   )r   r   s   "€r   r   ÚRwkv7TimeMixing.__annotate__é   s   ø€ ÷ <
ñ <
™Yð <
±3ñ <
r!   c                ó  <€ \         SV `  4        W n        Wn        VP                  V n        VP
                  V n        V P                  V P
                  ,          V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        \        4       V n        \        P                  ! ^^V P                  34      V n        \        P                  ! ^^V P                  34      V n        \        P                  ! ^^V P                  34      V n        \        P                  ! ^^V P                  34      V n        \        P                  ! ^^V P                  34      V n        \        P                  ! ^^V P                  34      V n        \        P                  ! V P                  V P
                  34      V n        \        P                  ! V P                  V P
                  34      V n        \        P                  ! V P                  V P
                  34      V n        \0        P2                  ! V P                  V P                  RR7      V n        \0        P2                  ! V P                  V P                  RR7      V n        \0        P2                  ! V P                  V P                  RR7      V n        \0        P2                  ! V P                  V P                  RR7      V n        \=        V P
                  V P                  RR7      V n        \A        V P                  V P                  V P                  RR7      V n!        V P                  ^ 8”  d3   \A        V P                  V P                  V P                  RR7      V n"        \A        V P                  V P                  V P                  RR7      V n#        \A        V P                  V P                  V P                  RRR	7      V n$        R# )
r3   FrŒ   gñhãˆµøD?©rr   r‰   )r„   r…   Nrˆ   )r„   r…   rq   )%rm   rn   r¼   r¦   r   r   rt   r   r   r   r   rœ   r®   r8   ro   Úx_rÚx_wr­   Úx_vÚx_aÚx_gÚk_kÚk_aÚr_kr�   r“   Úr_projÚk_projÚv_projÚo_projrk   Úg_normr€   Úw_loraÚv_loraÚa_loraÚg_lora©rs   r¦   r¼   ru   s   &&&€r   rn   ÚRwkv7TimeMixing.__init__é   sã  ø€ Ü‰ÑÔØ"ŒØŒ	Ø×+Ñ+ˆÔØŸ™ˆŒØ×)Ñ)¨T¯]©]Õ:ˆŒØ"×1Ñ1ˆÔØ"×1Ñ1ˆÔØ!%×!7Ñ!7ˆÔØ"&×"9Ñ"9ˆÔä%›<ˆÔä—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒÜ—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒÜ—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒÜ—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒÜ—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒÜ—8’8˜Q  4×#3Ñ#3Ð4Ó5ˆŒä—8’8˜TŸ^™^¨T¯]©]Ð;Ó<ˆŒÜ—8’8˜TŸ^™^¨T¯]©]Ð;Ó<ˆŒÜ—8’8˜TŸ^™^¨T¯]©]Ð;Ó<ˆŒä—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒÜ—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒÜ—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒÜ—i’i × 0Ñ 0°$×2BÑ2BÈÔOˆŒä& t§}¡}°d·n±nÈ%ÔPˆŒäØ×ÑØ×ÑØ×0Ñ0Øô	
ˆŒð �>‰>˜AÔÜØ× Ñ Ø× Ñ Ø!×0Ñ0Øô	ˆDŒKô Ø×ÑØ×ÑØ×,Ñ,Øô	
ˆŒô Ø×ÑØ×ÑØ×/Ñ/Ø Øô
ˆŽr!   c                ó¨  € VP                   w  r‰ p
VfD   \        P                  ! W€P                  V P                  V P                  3VP
                  R7      p\        P                  ! 4       \        P                  8X  d3   \        P                  P                  4       '       d   \        WW4WVV4      # . p\        V	4       F]  p\        VRV3,          VRV3,          VRV3,          VRV3,          VRV3,          VRV3,          V4      w  r×VP                  V4       K_  	  \        P                  ! V^R7      P                  VP
                  4      pW×3# )N)rf   r>   rž   )re   r8   ro   rt   r   rf   Údefault_deviceÚgpurQ   rR   rI   ÚrangerG   ÚappendÚstackÚastype)rs   r?   r@   rA   rB   rC   rD   rE   rh   r    Ú_ÚysÚtr/   s   &&&&&&&&      r   Ú_wkv7ÚRwkv7TimeMixing._wkv7'  s  € Ø—W‘W‰
ˆˆa�ØŠ=Ü—H’HØ—N‘N D§M¡M°4·=±=ÐAÈÏÉôˆEô ×ÒÓ¤"§&¡&Ô(¬R¯X©X×-BÑ-B×-DÒ-DÜ˜q Q¨1°Ó7Ð7àˆBÜ˜1–X�Ü)Ø�a˜�d•G˜Q˜q !˜t�W a¨¨1¨¥g¨q°°A°­w¸¸!¸Q¸$½ÀÀ1ÀaÀ4ÅÈ%ó‘�ð —	‘	˜!–ñ	 ô —’˜ !Ô$×+Ñ+¨A¯G©GÓ4ˆAØ�8ˆOr!   c           	     óä  € Vf   R R rTMV^ ,          V^,          rTVP                   w  rgpV P                  W4      p	W‘,
          p
\        WV P                  4      p\        WV P                  4      p\        WV P
                  4      p\        WV P                  4      p\        WV P                  4      p\        WV P                  4      pV P                  V4      P                  WgV P                  V P                  4      pV P                  V4      P                  WgV P                  V P                  4      pV P                  V4      P                  WgV P                  V P                  4      p\        P                   ! V P#                  V4      4      P                  WgV P                  V P                  4      pV P%                  V4      pV P&                  ^ 8X  d   TpM^\        P                   ! V P)                  V4      4      P                  WgV P                  V P                  4      p\        VVV,
          V4      p\        P                   ! V P+                  V4      P                  WgV P                  V P                  4      4      P-                  \        P.                  4      p\        P0                  ! RV,          4      P-                  VP2                  4      p\5        VV P6                  ,          4      pV^V^,
          V P8                  ,          ,           ,          pV) pVV,          pV P;                  VVVVVVV4      w  ppV P=                  VP                  WgV P                  V P                  4      4      pVVV,          V P>                  ,          PA                  RRR7      V,          ,           P                  WgV.4      pVe   VRRR 1R3,          V^ &   VV^&   V PC                  VV,          4      pVV3# )NTr4   r>   g]S ³³hã¿r7   )"re   r®   r1   rÁ   rÂ   r­   rÃ   rÄ   rÅ   rÊ   Úreshapert   r   rË   rÉ   r8   rˆ   rÐ   rÑ   r¼   rÏ   rÎ   rÚ   Úfloat32Úexprf   r<   rÆ   rÇ   rÞ   rÍ   rÈ   ÚsumrÌ   )rs   r.   Úv_firstr´   Útoken_shift_cacheÚstate_cacherh   r    r]   rµ   r¶   ÚxrÚxwÚxkÚxvÚxaÚxgr«   r¬   Ú
receptanceÚiclrÚgateÚvvÚdecayÚkkrC   rD   ÚoutÚnew_state_caches   &&&&                         r   rz   ÚRwkv7TimeMixing.__call__;  s  € ØŠ=Ø-1°4™{à-2°1­X°u¸Qµx˜{à—'‘'‰ˆˆaØ×!Ñ! !Ó7ˆØ�Zˆä�Q˜DŸH™HÓ%ˆÜ�Q˜DŸH™HÓ%ˆÜ�Q˜DŸH™HÓ%ˆÜ�Q˜DŸH™HÓ%ˆÜ�Q˜DŸH™HÓ%ˆÜ�Q˜DŸH™HÓ%ˆà�k‰k˜"‹o×%Ñ% a¨D¯N©N¸D¿M¹MÓJˆØ—‘˜B“×'Ñ'¨¨d¯n©n¸d¿m¹mÓLˆØ—[‘[ “_×,Ñ,¨Q°4·>±>À4Ç=Á=ÓQˆ
Ü�zŠz˜$Ÿ+™+ b›/Ó*×2Ñ2°1¸¿¹ÈÏÉÓWˆØ�{‰{˜2‹ˆà�>‰>˜QÔØ‰Gä—’˜DŸK™K¨›OÓ,×4Ñ4Ø�d—n‘n d§m¡móˆBô ˜E 7¨U¥?°BÓ7ˆEä—
’
Ø�K‰K˜‹O×#Ñ# A¨$¯.©.¸$¿-¹-ÓHó
ç
‰&”—‘Ó
ð 	ô —’�y 5Õ(Ó)×0Ñ0°×1AÑ1AÓBˆÜ�c˜DŸH™H•nÓ&ˆØ�Q˜$ �( d§h¡hÕ.Õ.Õ/ˆØˆCˆØ��Iˆà#Ÿz™zØ˜˜s E¨1¨a°ó 
Ñˆˆ_ð �k‰k˜#Ÿ+™+ a¨D¯N©N¸D¿M¹MÓJÓKˆà�: Õ# d§h¡hÕ.×3Ñ3¸ÀdÐ3ÓKÈeÕSÕSß
‰'�1˜�)Ó
ð 	ð ÒØ˜˜B™C ˜•|ˆE�!‰HØ&ˆE�!‰Hà�k‰k˜# �*Ó%ˆØ�Gˆ|Ðr!   )rÐ   r   r¦   r   rÑ   rÍ   r   r   r   rÇ   rÆ   rÊ   r¼   rt   rÌ   rÈ   rÉ   r®   rÏ   r   rË   rÎ   rÄ   rÅ   r­   rÁ   rÃ   rÂ   )
r#   r$   r%   r&   rn   rÞ   rz   r(   r)   r}   r~   s   @@r   r¹   r¹   è   s   ù‡ € ÷<
ó <
ò|÷(5ò 5r!   r¹   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )Ú
Rwkv7Layeris  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r»   r½   )r   r   s   "€r   r   ÚRwkv7Layer.__annotate__t  s   ø€ ÷ Jñ J™Yð J±3ñ Jr!   c                óÎ  <€ \         SV `  4        W n        V P                  ^ 8X  d2   \        P                  ! VP
                  VP                  R7      V n        \        WP                  R7      V n	        \        V4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP
                  VP                  R7      V n        R# )r   rÀ   ©r¼   N)rm   rn   r¼   r�   Ú	LayerNormr   r   Úpre_normr¹   Úattnr£   ÚffnÚ	attn_normÚffn_normrÒ   s   &&&€r   rn   ÚRwkv7Layer.__init__t  s�   ø€ Ü‰ÑÔØ"ŒØ�>‰>˜QÔÜŸLšL¨×)9Ñ)9¸t¿}¹}ÔMˆDŒMÜ# D·N±NÔCˆŒ	Ü% dÓ+ˆŒÜŸš d×&6Ñ&6¸D¿M¹MÔJˆŒÜŸš T×%5Ñ%5¸4¿=¹=ÔIˆŽr!   c                óð   € V P                   ^ 8X  d   V P                  V4      pV P                  V P                  V4      W#4      w  rBW,           pW@P	                  V P                  V4      V4      ,           pWR3# )r   )r¼   rþ   rÿ   r  r   r  )rs   r.   rå   r´   Úhrô   s   &&&&  r   rz   ÚRwkv7Layer.__call__~  sb   € Ø�>‰>˜QÔØ—‘˜aÓ ˆAà—Y‘Y˜tŸ~™~¨aÓ0°'ÓA‰
ˆØ�EˆØ—(‘(˜4Ÿ=™=¨Ó+¨UÓ3Õ3ˆØˆ|Ðr!   )rÿ   r  r   r  r¼   rþ   r|   r~   s   @@r   rø   rø   s  s   ù‡ € ÷Jó J÷ò r!   rø   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú
Rwkv7Modeliˆ  c                ó    <€ V ^8„  d   QhRS[ /# r¥   r§   )r   r   s   "€r   r   ÚRwkv7Model.__annotate__‰  s   ø€ ÷ Fñ F™Yñ Fr!   c                óX  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        R# u upi )rü   rÀ   N)rm   rn   r�   Ú	Embeddingr   r   Ú
embeddingsr×   r   rø   Úlayersrý   r   r;   )rs   r¦   Úiru   s   && €r   rn   ÚRwkv7Model.__init__‰  sy   ø€ Ü‰ÑÔÜŸ,š, t§¡¸×8HÑ8HÓIˆŒä38¸×9OÑ9OÔ3Pó
Ù3P¨aŒJ�t×)Ñ3Pñ
ˆŒô —L’L ×!1Ñ!1°t·}±}ÔEˆŽ	ùò
s   ÁB'c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r.   rX   )r   r   s   "€r   r   r
  ‘  s   ø€ ÷ ñ ™"Ÿ(™(ñ r!   c                óà   € V P                  V4      pVf   R .\        V P                  4      ,          pR p\        V P                  V4       F  w  rEV! WV4      w  rK  	  V P	                  V4      # r-   )r  Úlenr  Úzipr;   )rs   r.   r´   rå   ÚlayerÚcs   &&&   r   rz   ÚRwkv7Model.__call__‘  sd   € Ø�O‰O˜AÓˆØŠ=Ø�FœS §¡Ó-Õ-ˆEàˆÜ˜DŸK™K¨Ö/‰HˆEÙ˜q¨1Ó-‰JˆA‰wñ 0à�y‰y˜‹|Ðr!   )r  r  r;   r|   r~   s   @@r   r  r  ˆ  s    ù‡ € ÷Fó F÷÷ ð r!   r  c                   ó€   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       tR t	]R	 4       t
R
tVtV ;t# )ÚModeliœ  c                ó    <€ V ^8„  d   QhRS[ /# r¥   r§   )r   r   s   "€r   r   ÚModel.__annotate__�  s   ø€ ÷ Tñ T™Yñ Tr!   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# rª   )rm   rn   r¦   r   r  Úmodelr   r�   r“   r   r   Úlm_head)rs   r¦   ru   s   &&€r   rn   ÚModel.__init__�  sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rU   rX   )r   r   s   "€r   r   r  ¥  s   ø€ ÷ ñ ™rŸx™xñ r!   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# r-   )r  r¦   r   r  Ú	as_linearr  )rs   rU   r´   r.   Úlogitss   &&&  r   rz   ÚModel.__call__¥  sU   € Ø�J‰J�vÓ%ˆà�9‰9×(×(Ð(Ø—Z‘Z×*Ñ*×4Ñ4°QÓ7ˆFð ˆð —\‘\ !“_ˆFàˆr!   c                ó|   € \        \        V P                  4      4       Uu. uF  p\        ^R7      NK  	  up# u upi )é   )Úsize)r×   r  r  r   )rs   rÛ   s   & r   Ú
make_cacheÚModel.make_cache¯  s/   € Ü-2´3°t·{±{Ó3CÔ-DÓEÑ-D¨” ×#Ñ-DÑEÐEùÒEs   ¡9c                ó.   € V P                   P                  # r-   )r  r  )rs   s   &r   r  ÚModel.layers²  s   € à�z‰z× Ñ Ð r!   c                ó   € VP                  4        Fy  w  r#R V9   g   RV9   g
   RV9   g   K  W,          P                  V P                  P                  V P                  P                  ,          V P                  P                  4      W&   K{  	  V# )rÆ   rÇ   rÍ   )Úitemsrá   r¦   r   r   )rs   ÚweightsrA   rB   s   &&  r   ÚsanitizeÚModel.sanitize¶  sj   € Ø—M‘M–O‰DˆAØ˜Œz˜U aœZ¨8°q®=Ø$�Z×/Ñ/Ø—I‘I×)Ñ)¨T¯Y©Y×-?Ñ-?Õ?ÀÇÁ×ASÑASó�“
ñ $ð
 ˆr!   c                ó   € R  pV# )c                 ó*   € R V 9   g   RV 9   d   R^/# R# )zlora.2r  ÚbitsTr"   )ÚpathrÛ   s   &&r   Ú	predicateÚ(Model.quant_predicate.<locals>.predicateÀ  s   € Ø˜4Ô <°4Ô#7Ø �{Ð"Ùr!   r"   )rs   r5  s   & r   Úquant_predicateÚModel.quant_predicate¾  s   € ò	ð
 Ðr!   )r¦   r  r  r   r-   )r#   r$   r%   r&   rn   rz   r(  Úpropertyr  r/  r7  r(   r)   r}   r~   s   @@r   r  r  œ  sS   ù‡ € ÷Tó T÷ò òFð ñ!ó ð!òð ñó ÷ð r!   r  ) Údataclassesr   Ú	functoolsr   Útypingr   Úmlx.coreÚcorer8   Úmlx.nnr�   Úbaser   r´   r   r	   Úcompiler1   r<   rG   rV   rg   rI   ÚModulerk   r€   rœ   r£   r¹   rø   r  r  r"   r!   r   Ú<module>rC     s-  ðõ "Ý Ý å Ý å Ý ð ô&�ó &ó ð&ñ 	ˆ�‰˜tÔ$ñó %ðñ 	ˆ�‰˜tÔ$ñKó %ðKð ‡�ñó ðòGñT !Ó"€õô4U�r—y‘yô Uô!;ˆ2�9‰9ô !;ôHA�—‘ô Aô2˜Ÿ™ô 2ô.H�b—i‘iô HôV�—‘ô ô*�—‘ô ô()ˆB�I‰Iö )r!   