+
    QV-j×$  ã                   óÔ   € R t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RIH	t	 ^ RI
Ht ^RIHt ]	P                  ! ]4      t ! R R]]P$                  4      t ! R R	4      tR
]n         R# )z+
CLI entry point for `transformers serve`.
N)Ú	Annotated)Úlogging)Úis_serve_available)Úset_torch_seedc                   ó"   € ] tR t^#tRtRtRtRtR# )ÚReasoningModeÚonÚoffÚauto© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚONÚOFFÚAUTOÚ__static_attributes__r   ó    Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/cli/serve.pyr   r   #   s   † Ø	€BØ
€CØ„Dr   r   c                   óˆ   a € ] tR t^)t o RRRRR]P
                  RRRRRRRRRRRRRRRR3V 3R lR	 lltR
 tR tR t	Rt
V tR# )ÚServeNFr
   i,  Ú	localhosti@  Úwarningc          2      ó  <€ V ^8„  d   QhRS[ S[R,          S[P                  ! RR7      3,          RS[ S[S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          R	S[ S[S[P
                  ! R
R7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[S[P
                  ! RR7      3,          RS[ S[S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! RR7      3,          RS[ S[R,          S[P
                  ! R R7      3,          R!S[ S[R,          S[P
                  ! R"R7      3,          R#S[ S[S[P
                  ! R$R7      3,          R%S[ S[S[P
                  ! R&R7      3,          R'S[ S[S[P
                  ! R(R7      3,          R)S[ S[S[P
                  ! R*R7      3,          R+S[ S[R,          S[P
                  ! R,R7      3,          R-S[ S[S[P
                  ! R.R/R07      3,          R1R/# )2é   Úforce_modelNz*Model to preload and use for all requests.)ÚhelpÚcontinuous_batchingzMEnable continuous batching with paged attention. Configure with --cb-* flags.Úattn_implementationz2Attention implementation (e.g. flash_attention_2).Úcompilez*Enable torch.compile for faster inference.Úquantizationz.Quantization method: 'bnb-4bit' or 'bnb-8bit'.Ú	reasoninguÀ   Reasoning mode. 'auto' uses the chat template default. Only applies to models that support reasoning via their chat template (e.g. Qwen3, Gemma 4) â€” for other models this flag has no effect.Úchat_template_kwargsz‰Default JSON kwargs forwarded to apply_chat_template (e.g. '{"enable_thinking": true}'); per-request chat_template_kwargs override these.Údevicez4Device for inference (e.g. 'auto', 'cuda:0', 'cpu').Údtypez2Override model dtype. 'auto' derives from weights.Útrust_remote_codezTrust remote code when loading.Úmodel_timeoutzGSeconds before idle model is unloaded. Ignored when force_model is set.Úcb_block_sizez6KV cache block size in tokens for continuous batching.Úcb_num_blocksz2Number of KV cache blocks for continuous batching.Úcb_max_batch_tokensz1Maximum tokens per batch for continuous batching.Úcb_max_memory_percentz/Max GPU memory fraction for KV cache (0.0-1.0).Úcb_use_cuda_graphz+Enable CUDA graphs for continuous batching.ÚhostzServer listen address.ÚportzServer listen port.Úenable_corszEnable permissive CORS.Ú	log_levelz'Logging level (e.g. 'info', 'warning').Údefault_seedzDefault torch seed.Únon_blockingTz1Run server in a background thread. Used by tests.)Úhiddenr   Úreturn)	r   ÚstrÚtyperÚArgumentÚboolÚOptionr   ÚintÚfloat)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚServe.__annotate__*   sG  ø€ ÷ cñ cá™s T�z©5¯>ª>Ð?kÔ+lÐlÕmðcñ 'ÙÙ�LŠLÐmÔnðpõ
ð	cñ 'Ù�$�J™ŸšÐ*^Ô_Ð_õ
ðcñ ™4¡§¢Ð3_Ô!`Ð`Õaðcñ  Ù�$�J™ŸšÐ*ZÔ[Ð[õ
ðcñ ÙÙ�LŠLð/ôðõ	
ðcñ2 (Ù�$�JÙ�LŠLðmôðõ
ð3cñD ™#™uŸ|š|Ð1gÔhÐhÕiðEcñF ™˜t�¡U§\¢\Ð7kÔ%lÐlÕmðGcñH %¡T©5¯<ª<Ð=^Ô+_Ð%_Õ`ðIcñJ !Ù‘—’Ð#lÔmÐmõ
ðKcñR !Ù�$�J™ŸšÐ*bÔcÐcõ
ðScñX !Ù�$�J™ŸšÐ*^Ô_Ð_õ
ðYcñ^ 'Ù�$�J™ŸšÐ*]Ô^Ð^õ
ð_cñd  )Ù�D�L™%Ÿ,š,Ð,]Ô^Ð^õ 
ðecñj %Ù�4�K™ŸšÐ+XÔYÐYõ
ðkcñr ™™UŸ\š\Ð/GÔHÐHÕIðscñt ™™UŸ\š\Ð/DÔEÐEÕFðucñv ™t¡U§\¢\Ð7PÔ%QÐQÕRðwcñx ™S¡%§,¢,Ð4]Ô"^Ð^Õ_ðycñz  ¡ d¥
©E¯LªLÐ>SÔ,TÐ TÕUð{cñ|  Ù‘%—,’, dÐ1dÔeÐeõ
ð}cðB 
ñCcr   c           
     óª  € \        4       '       g   \        R 4      h^ RIp^RIHp ^RIHp ^RIHp ^RI	H
p ^RIHp ^RIHp ^RIHp Ve   \#        V4       \$        P&                  ! R	4      pVP)                  \$        P*                  VP-                  4       ,          4       V! VV	V
VVVVR
7      V n        ^ RIHp  RVRVRVRVRV/P5                  4        U!U"u/ uF  w  p!p"V"f   K  V!V"bK  	  p#p!p"V#'       d	   V ! R/ V#B MRp$V! VVV$R7      V n        V'       dD   \8        P:                  ! V4      p\=        V\>        4      '       g   \@        PB                  ! R4      hM/ pV\D        PF                  8X  d   RVR&   MV\D        PH                  8X  d   RVR&   V! V P.                  V P6                  VR7      V n%        V! V P.                  V P6                  R7      V n&        V! V P.                  V P6                  VR7      V n'        V! V P.                  V P6                  4      V n(        V! V P.                  V PJ                  V PL                  V PN                  V PP                  V P6                  VR7      p%VPS                  V%VVRR7      p&VPU                  V&4      V n+        V'       d   V PY                  4        R# V PV                  P[                  4        R# u up"p!i )zRMissing dependencies for serving. Install with `pip install transformers[serving]`N)ÚChatCompletionHandler)ÚCompletionHandler)ÚModelManager)ÚResponseHandler)Úbuild_server)ÚTranscriptionHandler)ÚGenerationStateÚtransformers)r$   r%   r&   r   r!   r'   r   )ÚContinuousBatchingConfigÚ
block_sizeÚ
num_blocksÚmax_batch_tokensÚmax_memory_percentÚuse_cuda_graph)r   r    Ú	cb_configz,--chat-template-kwargs must be a JSON objectTÚenable_thinkingF)Úmodel_managerÚgeneration_stater#   )rQ   rR   )Úcompletion_handlerÚresponse_handlerÚtranscription_handlerrR   r/   Úinfo)r-   r.   r0   r   ).r   ÚImportErrorÚuvicornÚserving.chat_completionrA   Úserving.completionrB   Úserving.model_managerrC   Úserving.responserD   Úserving.serverrE   Úserving.transcriptionrF   Úserving.utilsrG   r   r   Ú
get_loggerÚsetLevelÚ
log_levelsÚlowerÚ_model_managerrH   rI   ÚitemsÚ_generation_stateÚjsonÚloadsÚ
isinstanceÚdictr6   ÚBadParameterr   r   r   Ú_chat_handlerÚ_completion_handlerÚ_response_handlerÚ_transcription_handlerÚConfigÚServerÚserverÚstart_serverÚrun)'Úselfr   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   rX   rA   rB   rC   rD   rE   rF   rG   Útransformers_loggerrI   ÚkÚvÚ	cb_kwargsrO   ÚappÚconfigs'   &&&&&&&&&&&&&&&&&&&&&&&                r   Ú__init__ÚServe.__init__*   s±  € ôD "×#Ò#ÜÐrÓsÐsãåBÝ9Ý7Ý5Ý0Ý?Ý2ð Ò#Ü˜<Ô(ô &×0Ò0°Ó@ÐØ×$Ñ$¤W×%7Ñ%7¸	¿¹Ó8IÕ%JÔKá*ØØØ/Ø 3Ø%Ø'Ø#ô
ˆÔõ 	:ð
 ˜mØ˜mØ"Ð$7Ø$Ð&;Ø Ð"3ð÷ ‰e‹gðô

ñ‘��1ð ô ˆAˆqŠDñð 	ñ 

÷ >GÑ,Ñ9¨yÒ9ÈDˆ	Ù!0Ø 3ØØô"
ˆÔ÷  Ü#'§:¢:Ð.BÓ#CÐ ÜÐ2´D×9Ò9Ü×(Ò(Ð)WÓXÐXð :ð $&Ð àœ×(Ñ(Ô(Ø6:Ð Ð!2Ò3Øœ-×+Ñ+Ô+Ø6;Ð Ð!2Ñ3á2Ø×-Ñ-Ø!×3Ñ3Ø!5ô
ˆÔñ $5Ø×-Ñ-Ø!×3Ñ3ô$
ˆÔ ñ
 "1Ø×-Ñ-Ø!×3Ñ3Ø!5ô"
ˆÔñ ';¸4×;NÑ;NÐPT×PfÑPfÓ&gˆÔ#áØ×ÑØ×ÑØ#×7Ñ7Ø!×3Ñ3Ø"&×"=Ñ"=Ø!×3Ñ3Ø#ô
ˆð —‘ ¨$°TÀV�ÓLˆØ—n‘n VÓ,ˆŒçØ×ÑÖà�K‰K�O‰OÖùóC

s   Ã	KÃ(Kc                ó„   a € V 3R  lp\         P                  ! VRRR7      S n        S P                  P                  4        R# )c                  ó°   <€ \         P                  ! 4       p \         P                  ! V 4       V P                  SP                  P                  4       4       R # )N)ÚasyncioÚnew_event_loopÚset_event_loopÚrun_until_completerr   Úserve)Úloopru   s    €r   Ú_runÚ Serve.start_server.<locals>._runÐ   s:   ø€ Ü×)Ò)Ó+ˆDÜ×"Ò" 4Ô(Ø×#Ñ# D§K¡K×$5Ñ$5Ó$7Ö8r   zuvicorn-threadF)ÚtargetÚnameÚdaemonN)Ú	threadingÚThreadÚ_threadÚstart)ru   r†   s   f r   rs   ÚServe.start_serverÏ   s2   ø€ õ	9ô
 !×'Ò'¨tÐ:JÐSXÔYˆŒØ�‰×ÑÖr   c                ó:   € V P                   P                  4        R# )z$Clear all loaded models from memory.N)rd   Úshutdown©ru   s   &r   Úreset_loaded_modelsÚServe.reset_loaded_modelsØ   s   € à×Ñ×$Ñ$Ö&r   c                ó0  € V P                   P                  4        V P                  P                  4        V P                  '       d!   V P                  P	                  4       '       g   R # RV P
                  n        V P                  P                  ^R7       R # )NT)Útimeout)rf   r‘   rd   r�   Úis_aliverr   Úshould_exitÚjoinr’   s   &r   Úkill_serverÚServe.kill_serverÜ   se   € Ø×Ñ×'Ñ'Ô)Ø×Ñ×$Ñ$Ô&Ø�|�|ˆ| 4§<¡<×#8Ñ#8×#:Ò#:ÙØ"&ˆ�‰ÔØ�‰×Ñ !ÐÖ$r   )rl   rm   rf   rd   rn   r�   ro   rr   )r   r   r   r   r   r   r|   rs   r“   rš   r   Ú__classdictcell__)r=   s   @r   r   r   )   s‹   ø‡ € ð quð
 ð Ødið ð ×Ñð ØlrØpvØchð ð ð ð ð ð àLWØIMØUZØbkØX\ð ÷Acò còJò'÷%ð %r   r   uð  
Run a FastAPI server to serve models on-demand with an OpenAI compatible API.
Models will be loaded and unloaded automatically based on usage and a timeout.


Endpoints:
    POST /v1/chat/completions â€” Chat completions (streaming + non-streaming).
    POST /v1/completions      â€” Legacy text completions from a prompt.
    GET  /v1/models           â€” Lists available models.
    GET  /health              â€” Health check.

Requires FastAPI and Uvicorn: pip install transformers[serving]
)Ú__doc__r€   Úenumrg   r‹   Útypingr   r6   Útransformers.utilsr   Útransformers.utils.import_utilsr   r_   r   r`   r   Úloggerr5   ÚEnumr   r   r   r   r   Ú<module>r¤      sa   ðñó Û Û Û Ý ã å &Ý >å )ð 
×	Ò	˜HÓ	%€ô�C˜Ÿ™ô ÷y%ñ y%ðx€†r   