+
    UV-j}	  ã                  ó”   € ^ RI Ht ^ RIHt ^ RIHt ^ RIHt ^ RI	t
^ RIHt ]! RRR7       ! R R	4      4       t ! R
 R4      tRR	.tR# )é    )Úannotations)Ú	dataclass)ÚPathN)ÚAutoTokenizerT)ÚfrozenÚslotsc                  ó,   € ] tR t^t$ R]R&   R]R&   RtR# )ÚTokenizerOutputzmx.arrayÚ	input_idsÚattention_mask© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotations__Ú__static_attributes__r   ó    Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/flux2/tokenizer.pyr
   r
      s   ‡ àÓØ×r   r
   c                  óN   € ] tR t^tRR R lltR R ltR R ltRR R	 lltR
tR# )ÚFlux2Tokenizerc               ó$   € V ^8„  d   QhRRRRRR/# )é   Ú
model_pathz
str | PathÚ
max_lengthÚintÚreturnÚNoner   )Úformats   "r   Ú__annotate__ÚFlux2Tokenizer.__annotate__   s!   € ÷ 
ñ 
 :ð 
¸3ð 
Èñ 
r   c                	óœ   € W n         \        P                  ! \        \	        V4      P                  4       R ,          4      RRR7      V n        R# )Ú	tokenizerT)Úlocal_files_onlyÚuse_fastN)r   r   Úfrom_pretrainedÚstrr   Ú
expanduserr#   )Úselfr   r   s   &&&r   Ú__init__ÚFlux2Tokenizer.__init__   s:   € Ø$ŒÜ&×6Ò6Ü”�ZÓ ×+Ñ+Ó-°Õ;Ó<Ø!Øô
ˆŽr   c               ó    € V ^8„  d   QhRRRR/# )r   Úpromptústr | list[str]r   z	list[str]r   )r   s   "r   r    r!      s   € ÷ 
ñ 
 oð 
¸)ñ 
r   c           
     	óò   € \        V\        4      '       d   V.M
\        V4      pV Uu. uF
  q3e   TMRNK  	  ppV Uu. uF'  pV P                  P	                  RRRV/.RRRR7      NK)  	  up# u upi u upi )NÚ ÚroleÚuserÚcontentFT)ÚtokenizeÚadd_generation_promptÚenable_thinking)Ú
isinstancer'   Úlistr#   Úapply_chat_template)r)   r-   ÚpromptsÚps   &&  r   Ú_format_promptsÚFlux2Tokenizer._format_prompts   s“   € Ü(¨´×5Ò5�6‘(¼4À»<ˆÙ7>Ó?±w°!š‘1¨2Ò-±wˆÐ?ñ ó
ñ �ð �N‰N×.Ñ.Ø˜& )¨QÐ/Ð0ØØ&*Ø %ð	 /ö ñ ñ
ð 	
ùò @ùò
s
   ©A/¿-A4c               ó    € V ^8„  d   QhRRRR/# )r   r-   r.   r   r   r   )r   s   "r   r    r!   '   s   € ÷ <ñ < ?ð <°sñ <r   c                	óú   € V P                  V4      p\        ;QJ d    R  V 4       F  '       d   K   RM	  RM! R  V 4       4      '       d   ^ # V P                  VRRRRR7      p\        R VR,           4       4      # )c              3  ó*   "  € T F	  qR 8H  x € K  	  R# 5i©r0   Nr   ©Ú.0r;   s   & r   Ú	<genexpr>Ú.Flux2Tokenizer.count_tokens.<locals>.<genexpr>)   ó   é € Ð(¡˜1�BŽw£ùó   ‚FTN)ÚpaddingÚ
truncationÚadd_special_tokensÚreturn_tensorsc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5i©N)Úlen)rC   Úidss   & r   rD   rE   2   s   é € Ð;Ñ': ”3�s—8�8Ó':ùs   ‚r   )r<   Úallr#   Úmax)r)   r-   r:   Útokenss   &&  r   Úcount_tokensÚFlux2Tokenizer.count_tokens'   sp   € Ø×&Ñ& vÓ.ˆß‹3Ñ(¡Ó(�3�3Š3Ñ(¡Ó(×(Ò(ÙØ—‘ØØØØ#Øð  ó 
ˆô Ñ; v¨kÖ':Ó;Ó;Ð;r   Nc               ó$   € V ^8„  d   QhRRRRRR/# )r   r-   r.   r   z
int | Noner   r
   r   )r   s   "r   r    r!   4   s$   € ÷ 
ñ 
Ø%ð
Ø3=ð
à	ñ
r   c           
     	óŒ  € V P                  V4      p\        ;QJ d    R  V 4       F  '       d   K   RM	  RM! R  V 4       4      '       d�   \        V4      p\        \        P
                  ! \        P                  ! V^ 3\        P                  R7      4      \        P
                  ! \        P                  ! V^ 3\        P                  R7      4      R7      # T P                  TRT;'       g    V P                  RRRR7      p\        \        P
                  ! VR,          4      \        P
                  ! VR	,          4      R7      # )
c              3  ó*   "  € T F	  qR 8H  x € K  	  R# 5irA   r   rB   s   & r   rD   Ú*Flux2Tokenizer.tokenize.<locals>.<genexpr>8   rF   rG   FT)Údtype)r   r   r   Únp)rH   r   rI   rJ   rK   r   r   )r<   rP   rN   r
   ÚmxÚarrayrZ   ÚemptyÚint32r#   r   )r)   r-   r   r:   ÚbatchrR   s   &&&   r   r4   ÚFlux2Tokenizer.tokenize4   sî   € ð ×&Ñ& vÓ.ˆß‹3Ñ(¡Ó(�3�3Š3Ñ(¡Ó(×(Ò(Ü˜“LˆEÜ"ÜŸ(š(¤2§8¢8¨U°A¨J¼b¿h¹hÔ#GÓHÜ!Ÿxšx¬¯ª°%¸°Ä2Ç8Á8Ô(LÓMôð ð —‘ØØ Ø!×4Ð4 T§_¡_ØØ#Øð  ó 
ˆô Ü—h’h˜v kÕ2Ó3ÜŸ8š8 FÐ+;Õ$<Ó=ô
ð 	
r   )r   r#   )i   rM   )	r   r   r   r   r*   r<   rS   r4   r   r   r   r   r   r      s   † ÷
õ
õ<÷
ó 
r   r   )Ú
__future__r   Údataclassesr   Úpathlibr   Úmlx.coreÚcorer[   ÚnumpyrZ   Útransformersr   r
   r   Ú__all__r   r   r   Ú<module>ri      sO   ðÝ "å !Ý å Û Ý &ñ �$˜dÔ#÷ð ó $ð÷
8
ñ 8
ðv Ð.Ð
/‚r   