
    ii             
          d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z( d d
l)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9m:Z: d Z;d Z< G d de!      Z=dee>   de?deee>      dee@   de=f
dZAde#de#deBfdZCdEdeeD   de"eD   fd ZEd! ZF G d" d#      ZGe G d$ d%             ZHe G d& d'             ZIe G d( d)             ZJe G d* d+             ZKe G d, d-             ZLe G d. d/             ZMe G d0 d1             ZN G d2 d3      ZO G d4 d5      ZPd6 ZQd7 ZRde$ee@ef      fd8ZS G d9 d:      ZT G d; d<e      ZUeeUfd=e@d>e>fd?ZVeeUfd=e@d>e>d@ePfdAZWdB ZXeYdCk(  r eZdD        eX        yy)F    N)deque)	dataclassfield)BaseHTTPRequestHandlerThreadingHTTPServer)Path)Empty)Queue)Thread)
AnyCallableDictListLiteral
NamedTupleOptionalSequenceTupleUnion)scan_cache_dir   )__version__)BatchGeneratorgeneration_streamstream_generate)can_trim_prompt_cachemake_prompt_cachetrim_prompt_cache)make_logits_processorsmake_sampler)loadsharded_loadc                      t        j                         d   } t         dt         j                   dt        j                          d|  S )Narchitecture-)mxdevice_infor   platform)gpu_archs    b/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/server.pyget_system_fingerprintr+   0   s?    ~~/H]!BNN+1X->->-@,A8*MM    c                     dddddd}d}| D ]  }|j                         s|dk(  s n|dz  }  t        | d |       }| |d  j                         j                         }t	        |||   z        S )Ng    .A    eAr   )MGMBGB r   .)isdigitfloatstripupperint)xsizessplitxidigitssizes         r*   
parse_sizer@   5   s    Cs#1=EE

c	
  1Ve9FefI$$&Dvd#$$r,   c                   ,    e Zd ZU eed<   eed<   eed<   y)StopConditionstop_mettrim_lengthtrim_text_lengthN)__name__
__module____qualname__bool__annotations__r9    r,   r*   rB   rB   A   s    Nr,   rB   tokenseos_token_idsstop_id_sequences
stop_wordsreturnc                    | r| d   |v rt        ddd      S t        ||      D ]Q  \  }}t        |       t        |      k\  s| t        |       d |k(  s1t        dt        |      t        |            c S  t        ddd      S )a  
    Determines whether the token generation should stop based on predefined
    conditions.

    Args:
        tokens (List[int]): The current sequence of generated tokens.
        eos_token_ids (set): The token IDs that represents the
          end-of-sequence. If the last token in ``tokens`` is in the set,
          the generation should stop.
        stop_id_sequences (List[List[[int]]): A list of integer lists, each
          representing a sequence of token IDs. If the end of the `tokens`
          list matches any of these sequences, the generation should stop.
        stop_words (List[str]): The stop words that correspond to the
            ``stop_id_sequences``.

    Returns:
        StopCondition: A named tuple indicating whether the stop condition has
          been met (`stop_met`) and how many tokens should be trimmed from the
          end if it has (`trim_length`) as well as the text that should be
          trimmed.
    Tr   )rC   rD   rE   NF)rB   ziplen)rL   rM   rN   rO   stop_ids	stop_words         r*   stopping_criteriarW   G   s    6 &*-dANN"#4jA)v;#h-'s8}n&'83$! #H%(^   B %QKKr,   s1s2c                      t        t               t                    }t         fdt        d|dz         D              S )z
    Checks if a suffix of s1 has overlap with a prefix of s2

    Args:
        s1 (Sequence): The first sequence
        s2 (Sequence): The second sequence

    Returns:
        bool: If the two sequences have overlap
    c              3   6   K   | ]  }| d  d | k(    y wNrK   ).0irX   rY   s     r*   	<genexpr>z#sequence_overlap.<locals>.<genexpr>}   s'     D*CQr1"#w"Ra& *C   r   )minrT   anyrange)rX   rY   max_overlaps   `` r*   sequence_overlapre   q   s5     c"gs2w'KD%;?*CDDDr,   messagesrole_mappingc                    dddddd}||n|}d}| D ]F  }|j                  |d   d      }|j                  d	d      }|j                  d
d      }|| | | z  }H ||j                  dd      z  }|j                         S )Nz}A chat between a curious user and an artificial intelligence assistant. The assistant follows the given rules no matter what.zASSISTANT's RULE: zUSER: zASSISTANT: 
)system_promptsystemuser	assistantstopr3   rolern   contentrm   )getrstrip)rf   rg   default_role_mappingpromptlinerole_prefixrn   rp   s           r*   convert_chatrw      s    O '"	 $0#;<AULF"&&tF|R8+((9b)[M'4&11	  l{B//F==?r,   c                    | D ]  }|j                  dd      }t        |t              rR|D cg c]  }|d   dk(  s|d    }}t        |      t        |      k7  rt	        d      dj                  |      |d<   n|d|d<   |j                  dd      x}s|D ]D  }|j                  d	d      x}s|j                  d
d      x}s-t        j                  |      |d
<   F  yc c}w )a  
    Convert message content to a format suitable for `apply_chat_template`.

    The function operates on messages in place. It converts the 'content' field
    to a string instead of a list of text fragments.

    Args:
        message_list (list): A list of dictionaries, where each dictionary may
          have a 'content' key containing a list of dictionaries with 'type' and
          'text' keys.

    Raises:
        ValueError: If the 'content' type is not supported or if 'text' is missing.

    rp   Ntypetextz&Only 'text' content type is supported.r3   
tool_callsFfunction	arguments)rq   
isinstancelistrT   
ValueErrorjoinjsonloads)	rf   messagerp   fragmenttext_fragmentsr{   	tool_callfuncargss	            r*   process_message_contentr      s      ++i.gt$1818XHV<LPV<V    >"c'l2 !IJJ!#!8GI_!#GI \599:9'	$==U;;4;#xxU;;t;,0JJt,<[) ( s
   C"C"c                       e Zd Ze G d d             Z G d d      Ze G d d             Zddedefd	Zd
 Z	e
d        Zd Zd Zd Zd ZddefdZddddee   dee   fdZd Zy)LRUPromptCachec                   (    e Zd ZU ee   ed<   eed<   y)LRUPromptCache.CacheEntryprompt_cachenbytesN)rF   rG   rH   r   r   rJ   r9   rK   r,   r*   
CacheEntryr      s    3ir,   r   c                   2    e Zd Zd Zd ZddefdZd Zd Zy)	LRUPromptCache.CacheOrderc                 @    t               | _        t               | _        y r\   )r   _lru_checkpoints_lruselfs    r*   __init__z"LRUPromptCache.CacheOrder.__init__   s    $)GD!DIr,   c                 X    t        | j                        t        | j                        z   S r\   )rT   r   r   r   s    r*   __len__z!LRUPromptCache.CacheOrder.__len__   s     tyy>C(=(=$>>>r,   
checkpointc                 ^    |r| j                   n| j                  }|j                  ||f       y r\   )r   r   append)r   modelrL   r   cs        r*   pushzLRUPromptCache.CacheOrder.push   s%    )3%%AHHeV_%r,   c                     	 | j                   j                  ||f       y # t        $ r  | j                  j                  ||f       Y y w xY wr\   )r   remover   r   )r   r   rL   s      r*   r   z LRUPromptCache.CacheOrder.remove   sD    >		  %1 >%%,,eV_=>s     &A	A	c                     t        | j                        t        | j                        k\  r| j                  j                         S | j                  j                         S r\   )rT   r   r   popleftr   s    r*   popzLRUPromptCache.CacheOrder.pop   sE    499~T%:%:!;;yy((**,,4466r,   NF)	rF   rG   rH   r   r   rI   r   r   r   rK   r,   r*   
CacheOrderr      s"    	 	?	&$ 	&	>	7r,   r   c                   R    e Zd ZU eed<   ee   ed<   ee   ed<   ee   ed<   eed<   y)LRUPromptCache.SearchResultr   exactshorterlongercommon_prefixN)rF   rG   rH   r   rJ   r   r9   rK   r,   r*   SearchResultr      s+    
CycS	r,   r   max_size	max_bytesc                 f    || _         || _        i | _        | j                         | _        d| _        y Nr   )r   r   _cacher   r   _n_bytes)r   r   r   s      r*   r   zLRUPromptCache.__init__   s,     "OO%	r,   c                 ,    t        | j                        S r\   )rT   r   r   s    r*   r   zLRUPromptCache.__len__   s    499~r,   c                     | j                   S r\   )r   r   s    r*   r   zLRUPromptCache.nbytes   s    }}r,   c                 `   || j                   vr| j                  |dddd      S | j                   |   }d}d}|t        |      k  r0||   |v r)|||      }d|v r|}|dz  }|t        |      k  r||   |v r)|t        |      dz
  k(  r| j                  ||ddd      S d}|dkD  r|d|dz    }d}|}|dkD  rhd}	|g fg}
|
rW|
j                         \  }}d|v r|	t        |      t        |	      k  r$|}	n!|D ]  }|
j	                  ||   ||gz   f        |
rW|d| |	z   }| j                  |d|||      S )zASearch the cache for a prompt cache. Return exact or close match.Nr   rR   cacher   )r   r   rT   r   r   )r   r   rL   currentlast_cache_indexindexr   r   r   beststackextratoks                r*   _searchzLRUPromptCache._search   s   #$$UD$a@@++e$c&k!fUmw&>fUm,G'!#( QJE	 c&k!fUmw&> s6{Q.$$UFD$BB a3/!34G 19Dr]OE!&g%|s5zCI'=$&gclESEM%BC  '  FU^d*F  gv}MMr,   c                 B    | j                   |   }|D ]  }||   }	 |d   S )Nr   )r   )r   r   rL   r   r   s        r*   _getzLRUPromptCache._get  s.    ++e$CclG wr,   c                 H   | j                   |   g}|D ]  }|j                  |d   |           |d   d   j                  }| xj                  |z  c_        |d   d= t	        t        t        |                  D ]'  }||   ||dz      ||   }	}}t        |      dkD  r y ||	= ) y )NrR   r   r   r   )r   r   r   r   reversedrc   rT   )
r   r   rL   pathr   cache_bytesr^   d_prevdts
             r*   _deletezLRUPromptCache._delete  s    E"#CKKR& 2hw'..$HW%F,-A7DQKqAF1vzq		 .r,   c                 <   | j                  ||      }|j                  G| j                  |j                  |j                        }t	        j
                  |j                        g fS |j                  t        |j                        nd}|j                  |j                  |kD  r| j                  |j                  |j                        }t        |j                        rlt	        j
                  |j                        }t        t        |      dz
  |j                        }t        |j                        |z
  }t        ||       |||d  fS |dkD  rJ| j                  |j                  |j                        }t	        j
                  |j                        ||d  fS d |fS )Nr   r   )r   r   r   r   copydeepcopyr   r   rT   r   r   r   ra   r   )	r   r   rL   resultcache_entryshort_lengthr   prefixnum_to_trims	            r*   fetch_nearest_cachez"LRUPromptCache.fetch_nearest_cache+  sK   eV,<<#))FLL&,,?K==!9!9:B>>.4nn.Hs6>>*a==$)=)=)L))FLL&--@K$[%=%=>k&>&>?S[1_f.B.BC!&--069!%5fVWo--!))FLL&..AK==!9!9:F<=<QQQV|r,   r   c                 
   t        |      }|| j                  vri | j                  |<   | j                  |   }t        |      D ]]  \  }}||vri ||<   |rHd|v rD| xj                  |d   j                  z  c_        |d= | j
                  j                  ||d |        ||   }_ d|v r| j
                  j                  ||       n<t        d |D              }	| j                  ||	      |d<   | xj                  |	z  c_        | j
                  j                  |||       t        | j
                        | j                  kD  r/| j
                  j                         \  }}| j                  ||       | j                  | j                  kD  r|t        | j
                        dkD  rc| j
                  j                         \  }}| j                  ||       | j                  | j                  kD  rt        | j
                        dkD  ray y y y )Nr   c              3   4   K   | ]  }|j                     y wr\   )r   r]   r   s     r*   r_   z.LRUPromptCache.insert_cache.<locals>.<genexpr>S  s     =1ahh   r   r   )r   r   	enumerater   r   r   r   sumr   r   rT   r   r   r   r   )
r   r   rL   r   r   is_trimmabler   r^   r   r   s
             r*   insert_cachezLRUPromptCache.insert_cacheA  s   ,\:#!#DKK++e$'FAs'!!7 2!1!8!88G$		  r
3clG ( gIIUF+===K#|[IGGMM[(M		uf<tyy>DMM) IIMMOME6LL'mmdnn,TYY!1C IIMMOME6LL' mmdnn,TYY!1C,1C,r,   N)n_sequencesn_bytesr   r   c                   |t        d|      nd}|t        d|      nd}t        | j                        |kD  rH| j                  j                         \  }}| j	                  ||       t        | j                        |kD  rH| j
                  |kD  r@| j                  j                         \  }}| j	                  ||       | j
                  |kD  r?y y )Nr               )maxrT   r   r   r   r   )r   r   r   r   rL   s        r*   trim_tozLRUPromptCache.trim_to_  s     .9-Dc![)'%,%8#a/g$))n{* IIMMOME6LL' $))n{* mmg% IIMMOME6LL' mmg%r,   c           	      
   t        |       | j                  }}t        | j                  j                        dkD  r%t        | j                  j                  d   d         nd}t	        j
                  d| d|dz  dd| d	       y )
Nr   rR   r   zKV Caches: z seq, r.   z.2fz GB, latest user cache z tokens)rT   r   r   r   logginginfo)r   ncachesr   ntoks       r*   log_cache_statszLRUPromptCache.log_cache_statsl  s    d)T[[ 499--.2 		**2.q12 	
 	'&C(88OPTvU\]	
r,   )
   r   r   )rF   rG   rH   r   r   r   r   r9   r   r   propertyr   r   r   r   r   rI   r   r   r   r   rK   r,   r*   r   r      s      7 70    c   'NR ,(D (> /3T(&sm(=Ec](	
r,   r   c                   ,    e Zd ZU eed<   eed<   eed<   y)ModelDescriptionr   draftadapterN)rF   rG   rH   strrJ   rK   r,   r*   r   r   x  s    JJLr,   r   c                   J    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   y)SamplingArgumentstemperaturetop_ptop_kmin_pxtc_probabilityxtc_thresholdN)rF   rG   rH   r6   rJ   r9   rK   r,   r*   r   r     s#    LJLr,   r   c                   d    e Zd ZU eeeef      ed<   eed<   eed<   eed<   eed<   eed<   eed<   y)	LogitsProcessorArguments
logit_biasrepetition_penaltyrepetition_context_sizepresence_penaltypresence_context_sizefrequency_penaltyfrequency_context_sizeN)rF   rG   rH   r   r   r9   r6   rJ   rK   r,   r*   r   r     s:    c5j)**  r,   r   c                       e Zd ZU eed<   eed<   eed<   ee   ed<   e	ed<   e	ed<   e
ed<   e	ed<   ee	   ed	<   eeeef      ed
<   y)GenerationArgumentsr   samplinglogitsrO   
max_tokensnum_draft_tokenslogprobstop_logprobsseedchat_template_kwargsN)rF   rG   rH   r   rJ   r   r   r   r   r9   rI   r   r   r   rK   r,   r*   r  r    sQ    $$S	ON
3-"4S>22r,   r  c                   h    e Zd ZU ed   ed<   eed<   ee   ed<   eee      ed<   ee	eef      ed<   y)CompletionRequest)chatrz   request_typert   rf   toolsrg   N)
rF   rG   rH   r   rJ   r   r   r   r   r   rK   r,   r*   r  r    s>    .))K3iDI4S>**r,   r  c                       e Zd ZU eed<   eed<   eed<   eeegef   ed<   eed<   e	ed<   e	ed<   eed<   e
ed	<   eee	      ed
<   ee	   ed<   dZe	ed<   dZeed<   d Zy)GenerationContexthas_tool_callingtool_call_starttool_call_endtool_parserhas_thinkingthink_start_idthink_end_id	think_endrM   stop_token_sequencesrt   rR   prompt_cache_countF_should_stopc                     d| _         y NT)r  r   s    r*   rn   zGenerationContext.stop  s
     r,   N)rF   rG   rH   rI   rJ   r   r   r   r   r9   setr   r  r  rn   rK   r,   r*   r  r    sw    3*d*++NtCy/)I  L$!r,   r  c                   V    e Zd ZU eed<   eed<   eed<   ee   ed<   ee	ee
f      ed<   y)Responserz   tokenlogprobfinish_reason
top_tokensN)rF   rG   rH   r   rJ   r9   r6   r   r   r   r   rK   r,   r*   r"  r"    s.    
IJNC= d38n%%r,   r"  c                        e Zd ZddZd Zd Zy)
TimeBudgetc                     t         j                  j                         j                         dkD  | _        || _        || _        || _        d | _        d | _	        d| _
        d| _        y Nr   r   )r&   distributedinitr?   _is_distributed_budget_iterations_sync_frequency_start_current_iterations_loops_time_spent)r   budget
iterationssync_frequencys       r*   r   zTimeBudget.__init__  sZ    !~~22499;a?%-#' r,   c                 F    t        j                          | _        d| _        | S r   )timer1  r2  r   s    r*   __iter__zTimeBudget.__iter__  s    iik#$ r,   c                    | j                   s9t        j                         | j                  z
  | j                  kD  r
t	               y | xj
                  dz  c_        | j
                  | j                  kD  rH| xj                  dz  c_        | xj                  t        j                         | j                  z
  z  c_        | j                  | j                  z  dk(  rt        j                  t              5  t        j                  j                  | j                        j                         }d d d        t        j                  j!                         j#                         | j                  z  z  }| j                  |z  }t%        t'        | j                  |z        d      | _        d| _        d| _        t	               y # 1 sw Y   xY wr*  )r-  r9  r1  r.  StopIterationr2  r/  r3  r4  r0  r&   streamr   r+  all_sumitemr,  r?   r   round)r   	loop_timeavg_loop_timefactors       r*   __next__zTimeBudget.__next__  sR   ##yy{T[[(4<<7#o%  A% ##d&6&66KK1K		dkk 99{{T111Q6YY01 " 6 6t7G7G H M M OI 2 )NN'')..043G3GG! 5#&uT-=-=-F'G#K #$ /! 7 21s   38F>>GN)      ?   r   )rF   rG   rH   r   r:  rD  rK   r,   r*   r(  r(    s    	
"r,   r(  c                   4    e Zd Zdej                  fdZddZy)ModelProvidercli_argsc                 4   || _         d| _        d| _        d| _        d| _        d| _        t        j                  j                         }|j                         dkD  r|j                  r|nd| _        |j                         dkD  r|j                  s|nd| _        |j                         dkD  | _        i | _        | j                   j                  Kd| j                  | j                   j                  <   | j                  | j                   j                  d       yy)z@Load models on demand and persist them across the whole process.NFr   default_model)draft_model_path)rI  	model_keyr   	tokenizerdraft_modelis_batchabler&   r+  r,  r?   pipelinepipeline_grouptensor_groupis_distributeddefault_model_mapr!   )r   rI  groups      r*   r   zModelProvider.__init__  s     
!##%',zz|a'7H<M<MeSWZZ\A%h.?.?ET 	 $jjlQ. "$==*:ID""4==#6#67IIdmm))OIL +r,   Nc                 N   | j                   j                  ||      }| j                  |||fk(  r| j                  | j                  fS d | _        d | _        d | _        d | _        d| j                  j                  rdnd i}| j                  j                  r| j                  j                  |d<   |dk(  r| j                  j                  t        d      |xs | j                  j                  }| j                  r9t        | j                  j                  | j                  | j                        \  }nht        | j                  j                  ||      \  }nB| j                  r%t        || j                  | j                        \  }nt        |||      \  }| j                  j                   rj                  j"                  _        |||f| _        || _        | _        fd}|dk(  rF| j                  j
                  0t        | j                  j
                        \  | _        } ||       n"| |dk7  rt        |      \  | _        } ||       | j
                  *t%        d t'        | j                        D              | _        | j                  | j                  fS )	Ntrust_remote_codeTchat_templaterK  zEA model path has to be given as a CLI argument or in the HTTP request)adapter_pathtokenizer_configc                 d    | j                   j                   k7  rt        j                  d       y y )NzdDraft model tokenizer does not match model tokenizer. Speculative decoding may not work as expected.)
vocab_sizer   warning)draft_tokenizerrN  s    r*   validate_draft_tokenizerz4ModelProvider.load.<locals>.validate_draft_tokenizerF  s/    ))Y-A-AAE Br,   c              3   4   K   | ]  }t        |d         yw)mergeN)hasattrr   s     r*   r_   z%ModelProvider.load.<locals>.<genexpr>[  s      $-J7#-Jr   )rU  rq   rM  r   rN  rO  rI  rX  rY  r   rZ  rT  r"   rR  rS  r!   use_default_chat_templatedefault_chat_templateallr   rP  )	r   
model_pathrZ  rL  r[  r   r`  r_  rN  s	           @r*   r!   zModelProvider.load  sx   ++//
JG
>>j,8HII::t~~-- 
  )H)Hd
 ==&&040K0K_-(}}""* 6  (E4==+E+EL""#/MM'')<)<d>O>O$ y $(MM''!-%5$ y ""#/ 3 3T5F5F$ y $(!-%5$ y ==22&&.*3*I*I	'$l4DE
"	 /))504T]]5N5N0O-Do$_5).>/.Q045E0F-Do$_5# # $->tzz-J$ !D zz4>>))r,   )NN)rF   rG   rH   argparse	Namespacer   r!   rK   r,   r*   rH  rH    s    M!3!3 M.R*r,   rH  c                 L   t        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                  | j                  j                  |j                  |j                  d      g      S )Nri   )r   r   r   r   r   xtc_special_tokens)
r    r  r   r   r   r   r   r   eos_token_idencode)r   rN  s     r*   _make_samplerrn  b  sy    !!mm!!mm!!mm!!55mm11""T"
 r,   c           	      <   t        | j                  j                  | j                  j                  | j                  j                  | j                  j
                  | j                  j                  | j                  j                  | j                  j                        S r\   )	r   r  r   r   r   r   r   r   r  )r   s    r*   _make_logits_processorsrp  q  sj    !&&++$$))%%** r,   c                     |dk  ryt        j                  |  |dz
        }|d| j                         }| |   j                         }|j                  |      }t	        d t        |||      D              S )zDReturns info dicts for the top `top_logprobs` tokens from `logprobs`r   rK   r   )kthNc              3   0   K   | ]  \  }}}|||d   yw))idr#  r$  NrK   )r]   r^   sgs       r*   r_   z'_format_top_logprobs.<locals>.<genexpr>  s%      ;GAq! 1+;s   )r&   argpartitiontolistconvert_ids_to_tokenstuplerS   )r  r	  rN  sorted_indicestop_indicestxtss         r*   _format_top_logprobsr~  }  s    q__hYL14DEN ,/668KK(//1L**;7D ;l;  r,   c            
           e Zd ZdedefdZd Zd ZddZd Z	d	 Z
d
 Zd Zd Zd Zd Z	 ddededeeeegdf      fdZed        Zy)ResponseGeneratormodel_providerr   c                    || _         || _        t               | _        t	               | _        t        j                  j                         j                         dkD  | _
        t        j                  j                         j                         | _        d| _        t        | j                        | _        | j                   j#                          y )Nr   F)target)r  r   r
   requestsr(  _time_budgetr&   r+  r,  r?   r-  rank_rank_stopr   	_generate_generation_threadstart)r   r  r   s      r*   r   zResponseGenerator.__init__  s    ,(&L!~~22499;a?^^((*//1

"("?%%'r,   c                 F    d| _         | j                  j                          y r  )r  r  r   r   s    r*   stop_and_joinzResponseGenerator.stop_and_join  s    
$$&r,   c                 8    | j                   j                          y r\   )r  r   r   s    r*   r   zResponseGenerator.join  s    $$&r,   Nc                     d }| j                   r| j                  dk(  r:	 || j                  j                  |      }n| j                  j	                         }| j                  |      S # t
        $ r Y w xY w)Nr   timeout)r-  r  r  rq   
get_nowait
QueueEmpty_share_request)r   r  requests      r*   _next_requestzResponseGenerator._next_request  su    ##tzzQ&"mm///@G"mm668G ""7++  s   9A) )	A54A5c                    | j                   s|S t        j                  t              5  | j                  dk(  r|<t        j
                  t        j                  j                  d             	 d d d        y t        j                  t        j                  |            }t        j
                  t        j                  j                  |j                               t        j
                  t        j                  j                  |             |cd d d        S t        j                  j                  d      j                         }|dk(  r
	 d d d        y t        j                  |t        j                        }t        j                  j                  |      }t        j                  |      cd d d        S # 1 sw Y   y xY w)Nr   )dtype)r-  r&   r=  r   r  evalr+  r>  arraypickledumpsr?   r?  zerosuint8r   )r   objdatar?   s       r*   _share_objectzResponseGenerator._share_object  s$   ##JYY()zzQ;GGBNN22156	 *) 88FLL$56DGGBNN22499=>GGBNN22489 *) ~~--a055719 *) 88D9D>>11$7D!<<-# *))s    AF56BF53F5AF55F>c                     | j                   s|S ||dd  nd }| j                  |      }|y ||d   n	t               }|g|S r*  )r-  r  r
   )r   r  	shareablerqs       r*   r  z ResponseGenerator._share_request  sY    ##N#*#6GABKD	&&y1	".WQZEG~I~r,   c                 $   |j                   dk(  r|j                  }|j                  }|j                  }|j                  rt        |       |r!|j                  st        j                  d       | j                  j                  j                  }|j                  r+|j                         }|j                  |j                          |j                  |f|ddd|S |j!                  t#        ||            S |j!                  |j$                        S )Nr  zReceived tools but model does not support tool calling. If you think this is an error, file an issue here: https://github.com/ml-explore/mlx-lm/issuesT)r  add_generation_prompttokenize)r  rf   r  rg   has_chat_templater   r  r   r^  r  rI  chat_template_argsr  r   updateapply_chat_templaterm  rw   rt   )r   rN  r  r   rf   r  rg   r  s           r*   	_tokenizezResponseGenerator._tokenize  s   6)''HMME"//L**'1!;!;OOF &*%8%8%A%A%T%T",,);)@)@)B&&--d.G.GH4y44*.!	
 )  !''X|(LMM##GNN33r,   c           	          |j                   dk7  ry|j                  d   d   dk7  ryd}|j                  rFt        dt	        dt        |            dz
  d      D ]   }||    |j                  k(  s| dz
  } d|fS  d|fS )	Nr  )FrR   rR   ro   rl   r      T)r  rf   r  rc   ra   rT   r  )r   rN  r  rt   prompt_checkpointr^   s         r*   _compute_prompt_checkpointz,ResponseGenerator._compute_prompt_checkpoint  s    6)B'61
 !!1c"c&k2Q6:1":!9!99)*Q%&&& ;
 &&&r,   c                 L    | j                   j                  sy|j                  yy)NFT)r  rP  r
  )r   r   s     r*   _is_batchablezResponseGenerator._is_batchable   s$    ""//99 r,   c                 0   "#$ d }d }d }d #d }d}i "g $d $fd	}"fd}"# fd} j                   rt        j                  j                  t        j                  j
                  d         j                  t        j                        j                         }	t        j                  j                  |	        j                  sd }
|s|t        "      dkD  rd nd} ||      }
|
i|
\  }}
}|-||j                  k(  r j                  |      r	  j                  ||
|      }t#        j$                  |j&                  |j(                  |j*                  |j,                  |j.                  |j0                  |j2                  |j4                  |j6                  D cg c]  }|j9                  |d       c}|	      }|j!                  |        j:                  j=                           j:                  j?                  #|      \  }}t        |      t        |      z
  |_         |tC         jD                  j                        } jG                  ||
|      \  }}|jI                  |g|jJ                  |gtM        ||      gtO        |      g|g
      \  }||d d  ||jP                  |d"|<   ~ jD                  jR                  jT                  K jD                  jR                  jT                  }|jV                  } j:                  jY                  ||z
         g|	  jD                  j[                  |j                  j                  |j                  j\                  |j                  j^                        \  }} j                  |      s ja                  ||
|f       |j                  }|} jD                  jb                  #i "te        ||j4                   jR                  jf                   jR                  jh                   jR                  jj                  ||      }$jm                  ||
|f       d}$jm                  ||
|f       |Rt        "      dk(  r |rd }d }d }d #|jo                          d }d}g } jp                  D ]  }|js                         }|s no|D ]f  }"|jt                     }|d   jm                  |jv                         |jx                  dk7  r|d   j{                  |jv                         |d   j!                  t}        |d   j~                  |jv                  |j                  |jv                     j                         |jx                  t        |j                  j                  |                   |jx                  K|d   j!                  d         j:                  j                  #|d   |j:                         "|jt                  = |d   j                  sL|jm                  |jt                         i   j                  |      }|r{t        j                  t              5  |j                  |d      } | j                         D ]2  \  }}!|"vr"|   } j:                  j                  #|d   |!       "|= 4 	 d d d         j                  sy y # t        $ r}|j!                  |       Y d }~d }~ww xY wc c}w # t        $ r}|j!                  |       Y d }~Ld }~ww xY w# 1 sw Y   nxY w)NFc                 J    rj                         S j                  |       S r\   )r   r  )r  r   unprocessed_requestss    r*   get_next_requestz5ResponseGenerator._generate.<locals>.get_next_request  s&    #+//11))'22r,   c                 l    | D ].  \  }}}|v s|   d   j                  t        ||      |f       0 y )Nrqueue)putra   )r   uid	processedtotalbatch_resultss       r*   progress_callbackz6ResponseGenerator._generate.<locals>.progress_callback  s@    )-%Y-'!#&x044c)U6KU5ST *.r,   c                     | D ]@  \  }}}|   }|d   sj                   j                  |d   d |  t        |      d       B y )Nr   	cache_keyTr   )r   r   r   )promptsr  
prompt_endr   rsr  current_model_keyr   s        r*   checkpoint_callbackz8ResponseGenerator._generate.<locals>.checkpoint_callback  s_    *1&Z"3','!!..%{OLj[1K#	 / 	 +2r,   r   g?r  add_special_tokensr  r  r  r  r  r  r  r  rM   r  rt   )cachessamplerslogits_processorsprompt_checkpoints)ctxr  r  detokenizerr   )r   )stop_tokenscompletion_batch_sizeprefill_batch_sizeprefill_step_sizeprompt_progress_callbackprompt_checkpoint_callbackTr  rn   r  r  r  )return_prompt_cachesr\   )Jr-  r&   r+  r>  randomstateviewuint64r?  r
  r  rT   r   r  r  	Exceptionr  r  r  r  r  r  r  r  r  r  rM   rO   rm  r   r   r   r  r   r  r  insertr  rn  rp  r  rI  prompt_cache_bytesprompt_cache_nbytesr   r!   r   r   _serve_singlerM  r   decode_concurrencyprompt_concurrencyr  r   closer  nextr  r#  r%  	add_tokenr"  last_segmentr  r~  r	  r   r  r  r=  r   r   items)%r   current_modelcurrent_samplingcurrent_tokenizerbatch_generatordrain_batchr  r  r  r
  r  r  r  r   rt   erN  rV   r  r   restdo_checkpointcheckpoint_positionr  r  activer   uids_to_remove_	responsesrr   r  r   r  r  r  s%   `                                 @@@r*   r  zResponseGenerator._generate  s     !	3	U

	 >>))"))//!*<=BB299MRRTDIINN4 **G (3M8JQ8N  
 +7; "(/% $/%3**40!!%0A7D!Q
 ,)2)C)C(1(A(A&/&=&=$-$9$9%.%;%;'0'?'?"+"5"5%.%;%;&/&=&= .2__.-<	 &,,Y5,Q-<.  &C  JJsO%%557"&"3"3"G"G)6#KE4 .1[3t9-DC*} 1$2E2E2K2K L 77	7FS 7M#6 -33 %w"/i"@!A+B4+H*I,?+@ 4 FS  #%+AY"('0'<'<&3*M#& **33FFR $ 3 3 < < O O!0!D!D))11%&.1I
 %,!+/+>+>+C+C JJ,,djj.@.@$**BRBR,(y  --d3**FGT+BC $(JJM(1%(,(;(;(E(E%$&M&4$-$;$;.2mm.N.N+/==+K+K*.--*I*I1B3F'O )//$0GH
 #'K(//$0GH !,}%*"(,+/(,0),0)'--/*.&+!#**A / 4 4 6I$&!.quu!5{+22177;??f4"=1;;AGGDx(,,$ &} 5 B B ! !

177 3 8 8 : ! 4$%JJ0A0ACT!"
 ??6"8,006 --:: 16+3F !.aee 4!%=55*11!%%85 ' +B "&!3!3N!C!#45!0!7!7* "8 " 28-C"-7 (%23%7F --:: 16+3F !.c 2 2@	 6Q **, % !

1 !.h % !

1 !` 65sD   Z7 [#A[$ A\7	[ [[$	\	-\\	\c                 "   |\  }}fd}	 | j                   j                  }| j                   j                  }| j                   j                  }| j	                  |||      }t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  D cg c]  }|j!                  |d       c}|      }	j#                  |	       |j$                  )t&        j(                  j%                  |j$                         t+        ||      }
t-        |      }| j.                  j1                          | j.                  j3                  | j                   j4                  |      \  }}t7        |      t7        |      z
  |	_        |d d  }|Wt;        | j                   j                        }| j                   j                  "|t;        | j                   j                        z  }t=        ||||j>                  |
||||j@                  || jB                  jD                        D ]  }j#                  tG        |jH                  |jJ                  |jL                  |jJ                     jO                         |jP                  tS        |jL                  |jT                  |                   |jW                  |jJ                         |	jX                  s| jZ                  r
t]                n j#                  d        | j.                  j_                  | j                   j4                  ||       y c c}w # t`        $ r}j#                  |       Y d }~y d }~ww xY w)Nc                 ,    j                  | |f       y r\   )r  )tokens_processedtokens_totalr  s     r*   progressz1ResponseGenerator._serve_single.<locals>.progress  s    JJ(,78r,   Fr  r  )r   rN  rt   r  samplerr  r   rO  r  r  r  )1r  r   rN  rO  r  r  r  r  r  r  r  r  r  r  rM   rO   rm  r  r
  r&   r  rn  rp  r   r   r   rM  rT   r  r   r   r  r  rI  r  r"  rz   r#  r  r?  r%  r~  r	  r   r  r-  NotImplementedErrorr   r  )r   r  r   r  r   rN  rO  rt   rV   r  r  r  r   r  r  genr  r  s                    @r*   r  zResponseGenerator._serve_single  s    '	9X	''--E++55I--99K ^^Iw=F $!*!;!; ) 9 9'55%11&33(77#--&33'55 &*__&%4	 $$Y5$I%4& C  JJsO yy$		tyy) $D)4G 7 = --/++??##--vKE4 &)[3t9%<C"q	I})$*=*=*C*CD&&22>.t/B/B/N/NOOE '#??"3"'!%!6!6)1"&--"A"A 

		SYY/446)),LL$*;*;Y
   +##++133;> JJt **##--y%&F  	JJqMM	s2   CM* M%3HM* 	AM* %M* *	N3N		Nr  generation_argsr  c                     t               | j                  j                  ||f       fd}j                         }t	        |t
              r|| |       fS )Nc               3      K   	 j                         } | y t        | t              r| t        | t              r |   >|  Bwr\   )rq   r~   r  rz  )responser  response_queues    r*   _innerz*ResponseGenerator.generate.<locals>._innerM  sU     )--/#h	2"Nh.(4)84 s   AA)r
   r  r  rq   r~   r  )r   r  r  r  r  r  r   s      `  @r*   generatezResponseGenerator.generateD  sU     >7ODE	   "c9%IFH}r,   c                 .    | j                   j                  S r\   )r  rI  r   s    r*   rI  zResponseGenerator.cli_args`  s    ""+++r,   r\   )rF   rG   rH   rH  r   r   r  r   r  r  r  r  r  r  r  r  r  r  r   r   r9   r  r   rI  rK   r,   r*   r  r    s    
(} 
(N 
('',..
4>'$Y3v_J CG	" - $Hc3Z-=$>?	8 , ,r,   r  c                       e Zd Zdddedee   f fdZd Zd#defdZ	d#defd	Z
d
 Zd Zd Z	 	 	 	 	 	 	 	 d$dedeed   df   dee   dee   dee   deee      deeeeeef            deee      deee      dee   defdZdedee   fdZ	 	 	 d%dee   dee   dee   fdZdefdZdefdZd  Zd! Zd" Z xZS )&
APIHandlerN)system_fingerprintresponse_generatorr  c                    t        t        j                               | _        || _        |xs
 t	               | _        t        |   |i | y)z9
        Create static request specific metadata
        N)r9   r9  createdr  r+   r  superr   )r   r  r  r   kwargs	__class__s        r*   r   zAPIHandler.__init__f  sB     499;'"4"4"P8N8P$)&)r,   c                 p    | j                  dd       | j                  dd       | j                  dd       y )NzAccess-Control-Allow-Origin*zAccess-Control-Allow-MethodszAccess-Control-Allow-Headers)send_headerr   s    r*   _set_cors_headerszAPIHandler._set_cors_headersu  s5    6<7=7=r,   status_codec                 j    | j                  |       | j                  dd       | j                          y )NContent-typezapplication/jsonsend_responser  r  r   r  s     r*   _set_completion_headersz"APIHandler._set_completion_headersz  s-    ;');< r,   c                     | j                  |       | j                  dd       | j                  dd       | j                          y )Nr  ztext/event-streamzCache-Controlzno-cacher  r  s     r*   _set_stream_headerszAPIHandler._set_stream_headers  s=    ;')<=*5 r,   c                 F    | j                  d       | j                          y )N   )r  end_headersr   s    r*   
do_OPTIONSzAPIHandler.do_OPTIONS  s    $$S)r,   c                    | j                   | j                  | j                  d}| j                  |vr=| j                  d       | j	                          | j
                  j                  d       yt        | j                  d         }| j                  j                  |      }	 t        j                  |j                               | _        d}t!        j(                  dt        j$                  | j                  |              t+        | j                  t,              sJ dt/        | j                                | j                  j1                  dd      | _        | j                  j1                  dd      | _        | j                  j1                  dd      | _        | j                  j1                  dd      | _        | j                  j1                  d| j:                  j<                  j>                        | _        | j                  j1                  dd      | _         | j                  j1                  dd      | _!        | jB                  ?| j                  j1                  d| j:                  j<                  jB                        | _!        | j                  j1                  d| j:                  j<                  jD                        | _#        | j                  j1                  d| j:                  j<                  jH                        | _$        | j                  j1                  d| j:                  j<                  jJ                        | _%        | j                  j1                  d| j:                  j<                  jL                        | _&        | j                  j1                  dd      | _'        | j                  j1                  dd       | _(        | j                  j1                  d!d      | _)        | j                  j1                  d"d       | _*        | j                  j1                  d#d      | _+        | j                  j1                  d$d       | _,        | j                  j1                  d%d      | _-        | j                  j1                  d&d      | _.        | j                  j1                  d'd      | _/        | j                  j1                  d(d      | _0        | j                  j1                  d)d*      | _1        | j                  j1                  d+d      | _2        | j                  j1                  d,      | _3        | ji                          | j                  j1                  d-      }|xs g }t+        |tj              r|gn|} || j                            }| jm                  ||       y# t        j                  $ r}t!        j"                  d| d|j                                 | j                  d       | j	                          | j
                  j                  t        j$                  d	d
| i      j'                                Y d}~yd}~ww xY w).z:
        Respond to a POST request from a client.
        )z/v1/completionsz/v1/chat/completionsz/chat/completions  	   Not FoundNContent-LengthzJSONDecodeError: z - Raw body: i  errorzInvalid JSON in request body: 	zIncoming Request Body: indentz Request should be dict, but got r=  Fstream_optionsr   rK  rO  r  adaptersmax_completion_tokensr  r   r   r   r   r           r      r   r   r   r  r   r   r   r  r	  rR   r
  r  rn   )7handle_text_completionshandle_chat_completionsr   r  r  wfilewriter9   headersrfilereadr   r   decodebodyJSONDecodeErrorr   r"  r  rm  debugr~   dictry   rq   r=  r&  requested_modelrequested_draft_modelr  rI  r  r   r  tempr   r   r   r   r   r   r   r   r   r  r   r   r   r  r	  r
  r  validate_model_parametersr   handle_completion)r   request_factoriescontent_lengthraw_bodyr  r%  rO   r  s           r*   do_POSTzAPIHandler.do_POST  s   
  $;;$($@$@!%!=!=
 99--((-JJ\* T\\*:;<::??>2		

8??#45DI /

499V0T/UVWIIt
 	@-d499o->?	@ 

 iimmHe4"iimm,<dC#yy}}WoF%)YY]]=/%R" $		 7 7 @ @ Q Q!
 yy}}Z6))--(?F??""iimmd55>>IIDO  99==422;;@@
 YY]]7D,C,C,L,L,R,RS
YY]]7D,C,C,L,L,R,RS
YY]]7D,C,C,L,L,R,RS
"&))--0Dc"J'+yy}}5NPR'S$ $		.@# F%)YY]]3JB%O"!%/BC!H&*iimm4Lb&Q##yy}}->D!YY]]?C@))--d;		j%8 IIMM."=IIMM&$/	$(IIMM2H$I!&&( YY]]6*
%2
%/
C%@j\j
 /#DII.0w
3s ## 	MM-aShoo>O=PQR((-JJ

G'EaS%IJKRRT 	s   $(V! !Y	4BYY	c                 ~	   t        | j                  t              st        d      t        | j                  t
              r| j                  dk  rt        d      t        | j                  t        t
        f      r| j                  dk  rt        d      t        | j                  t        t
        f      r| j                  dk  s| j                  dkD  rt        d      t        | j                  t
              r| j                  dk  rt        d      t        | j                  t        t
        f      r| j                  dk  s| j                  dkD  rt        d      t        | j                  t
              r| j                  dk  rt        d	      t        | j                  t        t
        f      r| j                  dk  rt        d
      t        | j                  t
              r| j                  dk  rt        d      t        | j                  t        t
        f      st        d      t        | j                  t
              r| j                  dk  rt        d      t        | j                   t        t
        f      st        d      t        | j"                  t
              r| j"                  dk  rt        d      t        | j$                  t              st        d      | j&                  dk7  r0d| j&                  cxk  rdk  sn t        d| j&                  d      | j(                  bt        | j(                  t*              st        d      	 | j(                  j-                         D ci c]  \  }}t        |      | c}}| _        t        | j.                  t              r!d| j.                  cxk  rdk  st        d       t        d      t        | j0                  t              r!d| j0                  cxk  rdk  st        d       t        d      t        | j2                  t4              st        d      | j6                  %t        | j6                  t4              st        d      | j8                  &t        | j8                  t
              st        d      yyc c}}w # t        $ r t        d      w xY w)zg
        Validate the model parameters passed in the request for the correct types and values.
        zstream must be a booleanr   z)max_tokens must be a non-negative integerz(temperature must be a non-negative floatr   z%top_p must be a float between 0 and 1z$top_k must be a non-negative integerz%min_p must be a float between 0 and 1z/num_draft_tokens must be a non-negative integerz/repetition_penalty must be a non-negative floatz6repetition_context_size must be a non-negative integerz(Presence penalty must be must be a floatz4presence_context_size must be a non-negative integerz5frequency_context_size must be a non-negative integerzlogprobs must be a booleanrR   r   z.top_logprobs must be between 1 and 10 but got ,Nz)logit_bias must be a dict of int to floatr)        ?z5xtc_probability must be a float between 0.00 and 1.00rE  z2xtc_threshold must be a float between 0.00 and 0.5zmodel must be a stringzadapter must be a stringzseed must be an integer)r~   r=  rI   r   r  r9   r   r6   r   r   r   r  r   r   r   r   r   r  r  r	  r   r6  r  r   r   r7  r   r   r
  )r   kvs      r*   r:  z$APIHandler.validate_model_parameters  s    $++t,788$//3/4??Q3FHII$**UCL9T=M=MPQ=QGHH$**ucl3tzzA~VWDEE$**c*djj1nCDD$**ucl3tzzA~VWDEE$//59N9NQR9RNOO 422UCLA&&*NOO477=++a/UVV$//%>GHH455s;))A-STT$005#,?GHH466<**Q.TUU$--.9::"A0A0A,GR,G@ARARST@UV  ??&doot4 !LMMN9=9N9N9P"Q9PA3q6199P"Q t++U3,,44TVV 5TVVt))51dd>P>P6XTX6XQSS 7YQSS$..4566<<#Jt||S,I78899 DIIs)C677 *D ! #R N !LMMNs   R' -R!	R' !R' 'R<rz   r%  )lengthrn   prompt_token_countcompletion_token_countr  token_logprobsr&  rL   r{   reasoning_textrP   c           	      $   |xs g }|xs g }|	xs g }	| j                   | j                  | j                  | j                  | j                  d|dgd}|r.d|D cg c]  }|rt        |d   |      ni  c}i|d   d   d<   n7|r5dt        ||      D cg c]  \  }}t        ||       c}}i|d   d   d<   | j                  sHt        |t              rt        |t              st        d	      ||||z   d
|d<   ||dk\  r
d|i|d   d<   |d   d   }| j                  j                  d      r| j                  rdnd}d||
|	d||<   |S | j                  dk(  r|j                  |       |S t        d| j                         c c}w c c}}w )a  
        Generate a single response packet based on response type (stream or
        not), completion type and parameters.

        Args:
            text (str): Text generated by model
            finish_reason (Union[Literal["length", "stop"], None]): The reason the
              response is being sent: "length", "stop" or `None`.
            prompt_token_count (Optional[int]): The number of tokens in the prompt,
              used to populate the "usage" field (not used when stream).
            completion_token_count (Optional[int]): The number of tokens in the
              response, used to populate the "usage" field (not used when stream).
            prompt_cache_count (Optional[int]): The portion of prompt_token_count
              that was found in the cache when servicing the request.
            token_logprobs (Optional[List[float]]): The log probabilities per token,
              in token order.
            top_tokens (Optional[List[Tuple[Dict[str, Any]]]]): List of outputs from
              _format_top_logprobs, giving info on the top N tokens at each token position.
            tokens (Optional[List[int]]): List of tokens to return with logprobs structure
            tool_calls (Optional[List[str]]): List of tool calls.
            reasoning_text (Optional[str]): The reasoning text generated by the model.

        Returns:
            dict: A dictionary containing the response, in the same format as
              OpenAI's API.
        r   )r   r%  )rt  r  objectr   r	  choicesrp   )r	  rL  r  )rt  r$  z8Response type is complete, but token counts not providedprompt_tokenscompletion_tokenstotal_tokensusagecached_tokensprompt_tokens_detailschat.completiondeltar   rm   )ro   rp   	reasoningr{   text_completion)rz   zUnsupported response type: )
request_idr  object_typer7  r	  r6  rS   r=  r~   r9   r   
startswithr  )r   rz   r%  rF  rG  r  rH  r&  rL   r{   rI  r	  r  r^   rv  choicekey_names                    r*   generate_responsezAPIHandler.generate_response+  s   N (-2!'R%2
 //"&"9"9&&))|| %2
 EQEQ!D1A.;\2HY":.
 69&.6Q6QdaDAq)6Q2HY":. {{-s35s; N 
 "4%; 25K K!HW
 "-2D2I#%7>!"9: )$Q' &&'89"&++w9H#+(	 F8  !22MMtM$  :4;K;K:LMNNYs   FFr  rO   c                     t        t         j                   j                   j                        t         j                   j                   j                   j                   j                   j                        t         j                   j                   j                   j                    j"                   j$                   j&                        | j(                   j*                   j,                   j.                   j0                   j2                  
      } fd}	  j4                  j7                  ||      \  } jH                  r7 jK                  d
        j=                          tM        jN                  d       n& j;                  d
       tM        jN                  d       d}d}g }	d}
d fdfd}d}jP                  rftS        tU        jV                        dz
  dd      D ]@  }jV                  |   jX                  k(  r n"jV                  |   jZ                  k(  s>d} n d}g g }g }d}d}d}|D ]  }tM        jN                  |j\                         |r,|j\                  j^                  k(  rd}n||j\                  z  }nj`                  r|j\                  jb                  k(  rd}d}n_|r?|j\                  jd                  k(  r|	jg                  |
       d}
d}n.|
|j\                  z  }
n||j\                  z  }||j\                  z  }jg                  |jh                         |j,                  r|jg                  |jj                         |j.                  dkD  r|jg                  |jl                         to        jp                  jr                  |      }|jt                  rP|rdnd}jw                          d	tU              |jx                  z
   |d	tU        |      |jz                  z
   }d} n̉ jH                  r|st}        fdjr                  D              r|s|	s|r{ j                  |d	 ||	      |      } j>                  jA                  dtC        jD                  |       djG                                 j>                  j                          d}d}g }	|j                  |j                  } |r|
r|	jg                  |
        jH                  rg j                  || ||	      |      } j>                  jA                  dtC        jD                  |       djG                                 j>                  j                           j                   j                  d   r j                  tU        jV                        tU              j                        } j>                  jA                  dtC        jD                  |       djG                                 j>                  j                           j>                  jA                  djG                                 j>                  j                          y	 j                  ||tU        jV                        tU              j                  ||| ||	      
      }tC        jD                  |      jG                         }d}tM        jN                  d tC        jD                  ||!               j                  d"t        tU        |                    j=                           j>                  jA                  |        j>                  j                          y	# t8        $ rj} j;                  d        j=                           j>                  jA                  tC        jD                  d| i      jG                                Y d	}~y	d	}~ww xY w)#a  
        Generate a response to a prompt and send it to the client in a single batch.

        Args:
            prompt (List[int]): The tokenized prompt.
            stop_words (List[str]): A list of stop words passed to the
                stopping_criteria function
        )r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   r  )
r   r  r  rO   r  r  r  r	  r
  r  c                     t        j                  d|  d|        j                  rL	 j                  j	                  d|  d| dj                                j                  j                          y y # t        t        t        f$ r Y y w xY w)NzPrompt processing progress: /z: keepalive 

)
r   r   r=  r-  r.  rm  flushBrokenPipeErrorConnectionResetErrorOSError)processed_tokensrP  r   s     r*   keepalive_callbackz8APIHandler.handle_completion.<locals>.keepalive_callback  s    LL./?.@,P {{JJ$$&'7&8,tLSSU JJ$$&  ()=wG s   A
A6 6BB)r  r  r"  N   zStarting stream:zStarting completion:Fr3   r   c                     | j                  dd       xs t        t        j                               }t	        j
                  | d   d      | d<   | d|d}j                  r
|d<   dz  |S )	Nrt  r}   F)ensure_asciir|   )r|   ry   rt  r   r   )r   r   uuiduuid4r   r  r=  )r   tool_call_idoutr   tool_idxs      r*   format_tool_callz6APIHandler.handle_completion.<locals>.format_tool_call  sr    $==t4IDJJL8IL%)ZZ+&U&Ik" &""C
 {{'GAJr,   c                     | sg S g }| D ]`  }j                  |j                        }t        |t              r|j	                  fd|D               J|j                   |             b |S )Nc              3   .   K   | ]  } |        y wr\   rK   )r]   tcrp  s     r*   r_   zDAPIHandler.handle_completion.<locals>.parse_tools.<locals>.<genexpr>  s     !H2"22"6s   )r  r  r~   r   extendr   )r{   r   	tool_textparsedr  rp  r  s       r*   parse_toolsz1APIHandler.handle_completion.<locals>.parse_tools  sc    	F'	GMMBfd+MM!H!HHMM"26":; ( Mr,   r   rR   TrE  r{   rn   c              3   6   K   | ]  }t        |        y wr\   )re   )r]   sequencerL   s     r*   r_   z/APIHandler.handle_completion.<locals>.<genexpr>U  s       (@H ):(@r`   )r{   rI  zdata: ra  include_usagezdata: [DONE]

)rH  r&  rL   rI  r{   r#  zOutgoing Response: r$  r!  )Gr  r   r7  r8  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r	  r
  r  r  r  r  r  r  r-  r.  r   r  rm  r=  r  r   r5  r  rc   rT   rt   r  r  rz   r  r  r  r  r   r#  r$  r&  rW   rM   r  rC   rn   rD   rE   rb   r]  rb  r%  r&  completion_usage_responser  r  r   )r   r  rO   r   rg  r  r  in_tool_callmade_tool_callr{   ru  rw  in_reasoningr^   rI  rH  r&  rz   segmentr%  r  stop_conditionresponse_jsonr%  r  rp  rL   ro  s   ``                      @@@@r*   r;  zAPIHandler.handle_completion  s    #"**00
 ' ,,jjjjjj $ 4 4"00 ,??#'#:#:(,(D(D!%!6!6&*&@&@"&"8"8'+'B'B "!22]]**!%!:!:;
B	 
	 33<<"4 = MC ;;$$S)MM,-((-MM01 
		 
	 3szz?Q.B7::a=C$4$44ZZ]c&8&88#'L 8  
  !CMM#((# 88s}},#(L"chh.N%%#((c6I6I*I!%#88s000%%i0 "I#(L)I 388# MM#))$}}%%ckk2   1$!!#..1 /!!((	N &&0>F
 J#f+0J0J"JKIc$i.*I*IIJ{{< (+(@(@ 
n#55#.z#:'5	  6  H JJ$$vdjj.B-C4%H%O%O%QRJJ$$&%'N G!#J  , # 1 1O T Ii(;;--&z2-	 . H JJvdjj&:%;4@GGIJJJ"".43F3F3W99

OK**
 

  6$**X*>)?t!D!K!K!MN

  "JJ/6689JJ--CJJF&&-%-&z2 . H !JJx0779MFMM/

8F0S/TUV -s3}3E/FGJJ]+JJA  	((-JJTZZQC(9:AACD		s   )!]* *	_3A __c           
          | j                   | j                  d| j                  | j                  g ||||z   dd}||dk\  r
d|i|d   d<   |S )NrT  rM  )rt  r  rK  r   r	  rL  rQ  r   rR  rQ  rS  )rX  r  r7  r	  )r   rF  rG  r  r  s        r*   r{  z$APIHandler.completion_usage_response  st     //"&"9"9'))||!3%; 25K K
 ).@A.E!3:HW56 r,   c           	          | j                   }d|v sJ d       dt        j                          | _        | j                  rdnd| _        t        dd|d   |j                  d      xs d	|j                  d
            S )z
        Handle a chat completion request.

        Returns:
            mx.array: A mx.array of the tokenized prompt from the request body
        rf   z Request did not contain messagesz	chatcmpl-zchat.completion.chunkrT  r  r3   r  Nrg   )r3  rk  rl  rX  r=  rY  r  rq   )r   r3  s     r*   r,  z"APIHandler.handle_chat_completions  s     yyT!E#EE! &djjl^46:kk2GX HHW%HH^$
 	
r,   c                     dt        j                          | _        d| _        d| j                  v sJ d       t        d| j                  d   g dd      S )z
        Handle a text completion request.

        Returns:
            mx.array: A mx.array of the tokenized prompt from the request body
        zcmpl-rW  rt   z Request did not contain a promptrz   N)rk  rl  rX  rY  r3  r  r   s    r*   r+  z"APIHandler.handle_text_completions  s]     "$**,0,499$H&HH$ IIh
 	
r,   c                    | j                   j                  d      r| j                          y| j                   dk(  r| j                          y| j	                  d       | j                          | j                  j                  d       y)z9
        Respond to a GET request from a client.
        z
/v1/modelsz/healthr  r   N)r   rZ  handle_models_requesthandle_health_checkr  r  r-  r.  r   s    r*   do_GETzAPIHandler.do_GET  sf     99-&&(YY)#$$&((-JJ\*r,   c                     | j                  d       | j                          | j                  j                  dj	                                | j                  j                          y)z@
        Handle a GET request for the /health endpoint.
        rh  z{"status": "ok"}N)r  r  r-  r.  rm  rb  r   s    r*   r  zAPIHandler.handle_health_check  sJ     	$$S)

+2245

r,   c                 r   | j                  d       | j                          g d| j                  j                  d      }dt	        |      dkD  rdj                  |dd       fd}t               }|j                  D cg c]  } ||      s| }}|D cg c]  }|j                  d| j                  d }}| j                  j                  j                  rqt        | j                  j                  j                        }|j                         r8t        |j!                               }|j#                  |d| j                  d       d	|d
}	t%        j&                  |	      j)                         }
| j*                  j-                  |
       | j*                  j/                          yc c}w c c}w )zC
        Handle a GET request for the /v1/models endpoint.
        rh  )zconfig.jsonzmodel.safetensors.index.jsonztokenizer_config.jsonr`  N   c                    | j                   dk7  ryd| j                  vry| j                  k7  ry| j                  d   j                  D ch c]  }|j                  j
                   c}t        fdD              S c c}w )Nr   Fmainc              3   &   K   | ]  }|v  
 y wr\   rK   )r]   f
file_namess     r*   r_   zLAPIHandler.handle_models_request.<locals>.probably_mlx_lm.<locals>.<genexpr>  s     61qJs   )	repo_typerefsrepo_idfiles	file_pathnamerf  )repor  r  r  filter_repo_ids     @r*   probably_mlx_lmz9APIHandler.handle_models_request.<locals>.probably_mlx_lm   s{    ~~(TYY&)dlln.L48IIf4E4K4KL4Kq!++**4KLJ6666 Ms   Br   )rt  rK  r	  r   )rK  r  )r  r  r   r<   rT   r   r   reposr  r	  r  rI  r   r   existsr   resolver   r   r  rm  r-  r.  rb  )r   partsr  hf_cache_infor  downloaded_modelsmodelsrg  model_idr  r  r  r  s              @@r*   r  z APIHandler.handle_models_request  s    	$$S)X		$u:> XXeABi0N	7 '(*00
0TOD4ID0 	 
 *
 *	 ll!<<
 * 	 
 ""++11d55>>DDEJ  "z1134&")#'<< %f5

8,335

'

=


s   F/F/!F4)rh  )NNNNNNNN)NNN) rF   rG   rH   r  r   r   r   r  r9   r  r  r  r?  r:  r   r   r   r6   r   r   r   r6  r]  r  r;  r{  r,  r+  r  r  r  __classcell__)r  s   @r*   r  r  e  s   
 -1	*-* %SM	*>
!3 !
!s !N4`P8l -104,004<@&**.(,jj W%56<=j %SM	j
 !)j %SMj !e-j T%S#X"789j c#j T#Y'j !j 
jX@): @S	 @H -104,0	$SM !) %SM	2
): 
,
): 
&+8r,   r  hostportc                    | |f}t        j                  |t         j                  t         j                  d}t	        t        |            \  |_        }}}} ||fd      }t        j                  d       t        j                  d|  d| d       	 |j                          y # t        $ r# |j                          j                          Y y w xY w)N)ry   flagsc                  0     g| dt               i|S )Nr  )r+   )r   r  handler_classr  s     r*   <lambda>z"_run_http_server.<locals>.<lambda>;  s,    !
 !
57!
 	!
r,   z\mlx_lm.server is not recommended for production as it only implements basic security checks.zStarting httpd at z	 on port z...)socketgetaddrinfoSOCK_STREAM
AI_PASSIVEr  iteraddress_familywarningswarnr   r   serve_foreverKeyboardInterruptshutdownr  )	r  r  r  server_classr  server_addressinfosr  httpds	     ` `    r*   _run_http_serverr  -  s     D\N	f008I8IE <@U;L8LAq.	
E MM	4 LL%dV9TF#>?+ +((*+s   B( ()CCr  c                     t         j                  j                         }t        |j                  j
                        }t        ||      }|j                         dk(  rt        | ||       y |j                          y r   )
r&   r+  r,  r   rI  prompt_cache_sizer  r  r  r   )r  r  r  r  r  rV  r   r  s           r*   runr  N  s`     NN!E!."9"9"K"KLL*><Hzz|qt%78!r,   c                     t        j                  d      } | j                  dt        d       | j                  dt        d       | j                  dt        d	d
       | j                  dt        dd       | j                  dt        dd        | j                  dt        dd       | j                  ddd       | j                  dt        dg dd       | j                  dt        dd d!"       | j                  d#dd$       | j                  d%t
        d&d'       | j                  d(t
        d)d*       | j                  d+t        d,d-       | j                  d.t
        d&d/       | j                  d0t        d1d2       | j                  d3t        j                  d4d5       | j                  d6t        d7d8       | j                  d9t        d:d;       | j                  d<t        d=d>       | j                  d?t        d@dA       | j                  dBt        dC       | j                  dDddE       | j                         }t        j                  j                         r,t        j                         dF   }t        j                  |       t        j                   t#        t        |j$                  j'                         d       dGH       t)        |j*                  |j,                  t/        |             y )INzMLX Http Server.)descriptionz--modelz8The path to the MLX model weights, tokenizer, and config)ry   helpz--adapter-pathz9Optional path for the trained adapter weights and config.z--hostz	127.0.0.1z-Host for the HTTP server (default: 127.0.0.1))ry   defaultr  z--porti  z(Port for the HTTP server (default: 8080)z--draft-modelz,A model to be used for speculative decoding.)ry   r  r  z--num-draft-tokensz:Number of tokens to draft when using speculative decoding.r  z--trust-remote-code
store_truez)Enable trusting remote code for tokenizer)actionr  z--log-levelINFO)DEBUGr  WARNINGERRORCRITICALz%Set the logging level (default: INFO))ry   r  rL  r  z--chat-templater3   z)Specify a chat template for the tokenizerF)ry   r  r  requiredz--use-default-chat-templatezUse the default chat templatez--tempr)  z+Default sampling temperature (default: 0.0)z--top-prB  z-Default nucleus sampling top-p (default: 1.0)z--top-kr   z3Default top-k sampling (default: 0, disables top-k)z--min-pz5Default min-p sampling (default: 0.0, disables min-p)z--max-tokensi   z;Default maximum number of tokens to generate (default: 512)z--chat-template-argsznA JSON formatted string of arguments for the tokenizer's apply_chat_template, e.g. '{"enable_thinking":false}'z{}z--decode-concurrency    zFWhen a request is batchable then decode that many requests in parallelz--prompt-concurrency   zFWhen a request is batchable then process that many prompts in parallelz--prefill-step-sizei   z0Step size for prefill processing (default: 2048)z--prompt-cache-sizer   z@Maximum number of distinct KV caches to hold in the prompt cachez--prompt-cache-bytesz&Maximum size in bytes of the KV cachesz
--pipelinez,Use pipelining instead of tensor parallelism max_recommended_working_set_sizez)%(asctime)s - %(levelname)s - %(message)s)levelformat)rh  ArgumentParseradd_argumentr   r9   r6   r   r   r@   
parse_argsr&   metalis_availabler'   set_wired_limitr   basicConfiggetattr	log_levelr8   r  r  r  rH  )parserr   wired_limits      r*   r  r  ^  sg   $$1CDF
G  
 H  
 <	   7	   ;	   I	   8  
 A4   8   %,  
 :	   <	   B	   D	   J	   ZZ B	   U	   U	   ?	   O	   5  
 ;  
 D	xxnn&'IJ
;'gt~~335t<: 		499mD12r,   __main__z}Calling `python -m mlx_lm.server...` directly is deprecated. Use `mlx_lm.server...` or `python -m mlx_lm server ...` instead.r\   )[rh  r   heapqr   r   r  r(   r  r9  rk  r  collectionsr   dataclassesr   r   http.serverr   r   pathlibr   queuer	   r  r
   	threadingr   typingr   r   r   r   r   r   r   r   r   r   mlx.corecorer&   huggingface_hubr   _versionr   r  r   r   r   models.cacher   r   r   sample_utilsr   r    utilsr!   r"   r+   r@   rB   r9   r   r   rW   rI   re   r6  rw   r   r   r   r   r   r  r  r  r"  r(  rH  rn  rp  r~  r  r  r  r  r  rF   printrK   r,   r*   <module>r     s               ( C  %      * ! H H 
 ? %N
	%J 'LI'L'L DI'L S		'L
 'LTE Ex ED E4: Xd^ 0=D{
 {
|             3 3 3 + + + ! ! !( & & &%" %"Pj* j*Z	uT#s(^?T W, W,tE' EX %+
+
+J %"
"
" "" K3\ z		L 	F r,   