
    i{+                     H   d dl Z d dlZd dlZd dlmZ d dlZd dlZ	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ  ej2                  ej4                  d        ej6                  e      Z G d	 d
      Zd Zedk(  r	  ej@                   e              yy# e!$ r Y yw xY w)    N)generate)load)Model)AudioPlayer)
load_modelz4%(asctime)s - %(name)s - %(levelname)s - %(message)s)levelformatc                       e Zd Z	 	 	 	 	 	 	 	 	 	 ddZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zdedej                  fdZd Zy)VoicePipelinec                    || _         || _        || _        || _        || _        || _        || _        |	| _        |
| _        t        j                  |      | _        t        j                  d      | _        t        j                         | _        t        j                  d      | _        t        j"                         | _        y )N2   )maxsize)silence_thresholdsilence_durationinput_sample_rateoutput_sample_ratestreaming_intervalframe_duration_ms	stt_model	llm_model	tts_model	webrtcvadVadvadasyncioQueueinput_audio_queuetranscription_queueoutput_audio_queueLockmlx_lock)selfr   r   r   r   r   r   vad_moder   r   r   s              q/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_audio/sts/voice_pipeline.py__init__zVoicePipeline.__init__   s     "3 0!2"4"4!2"""==*!(r!:#*==? ")--";    c                    K   t         j                  d j                          t        j                   fd       d {   \   _         _        t         j                  d j                          t        j                   fd       d {    _        t         j                  d j                          t        j                   j                         _        y 7 7 Rw)NzLoading text generation model: c                  .    t         j                        S N)load_llmr   r"   s   r$   <lambda>z+VoicePipeline.init_models.<locals>.<lambda>:   s    HT^^,r&   zLoading text-to-speech model: c                  .    t         j                        S r)   )load_ttsr   r+   s   r$   r,   z+VoicePipeline.init_models.<locals>.<lambda>>   s    8DNN3Kr&   zLoading speech-to-text model: )loggerinfor   r   	to_threadllm	tokenizerr   ttsr   Whisperfrom_pretrainedsttr+   s   `r$   init_modelszVoicePipeline.init_models7   s     5dnn5EFG)0):):,*
 $
 $. 	4T^^4DEF **+KLL4T^^4DEF**4>>:$

 Ms%   >C%C!AC%C#AC%#C%c                 \  K   t        j                         | _        | j                          d {    t        j                  | j                               t        j                  | j                               t        j                  | j                               g}	 t        j                  |  d {    |D ]  }|j                           t        j                  |ddi d {    y 7 7 ;7 	# |D ]  }|j                           t        j                  |ddi d {  7   w xY ww)Nreturn_exceptionsT)
r   get_running_looploopr8   create_task	_listener_response_processor_audio_output_processorgathercancel)r"   tasksts      r$   startzVoicePipeline.startC   s     ,,.	     01 8 8 :; < < >?

	A..%(((
 ..%@4@@@ 	! ) A 
 ..%@4@@@sX   -D,C,A,D,C2 2C.3C2 7/D,&C0'D,.C2 0D,20D)"D%#D))D,c                    t        |t              rHt        j                  |t        j                        }|j                  t        j                        dz  }n|}|j                  t        j                        }t        j                  j                  |      t        j                  |j                        z  }|| j                  k  S )Ndtype      @)
isinstancebytesnp
frombufferint16astypefloat32linalgnormsqrtsizer   )r"   
audio_dataaudio_npenergys       r$   
_is_silentzVoicePipeline._is_silentV   s    j%(}}Zrxx@H

+g5  "H ??2::.)BGGHMM,BB....r&   c                     	 | j                   j                  || j                        S # t        $ r | j	                  |       cY S w xY wr)   )r   	is_speechr   
ValueErrorrX   )r"   frames     r$   _voice_activity_detectionz'VoicePipeline._voice_activity_detectione   sE    	.88%%eT-C-CDD 	.u---	.s   %( AAc                 $  K   t        | j                  | j                  dz  z        }t        j                  | j                  |dd| j
                        }|j                          t        j                  d       g }d}t        | j                  dz  | j                  z        }d}	 	 | j                  j                          d {   }| j                  |      }|rwd	}d}|j                  |       t        | d
      r&| j                  r| j                   j#                          | j$                  j'                  | j(                  j*                         nS|rQ|dz  }|j                  |       ||kD  r6|r.t        j                  d       | j-                  |       d {    g }d}d}7 7 # t.        j0                  t2        f$ r" |j5                          |j7                           w xY w# |j5                          |j7                          w xY ww)Ng     @@   rN   )
samplerate	blocksizechannelsrH   callbackzListening for voice input...r   i  FTcurrent_tts_taskzProcessing voice input...)intr   r   sdInputStream_sd_callbackrE   r/   r0   r   r   getr]   appendhasattrrd   current_tts_cancelsetr<   call_soon_threadsafeplayerflush_process_audior   CancelledErrorKeyboardInterruptstopclose)	r"   
frame_sizestreamframessilent_framesframes_until_silencespeaking_detectedr\   rZ   s	            r$   r>   zVoicePipeline._listenerl   s    //43I3IF3RST
-- &&
 	23"!!D(4+A+AA 
 "%	"4488:: ::5A	(,%$%MMM%( t%78T=R=R//335 II224;;3D3DE&!Q&MMM%($';;!"KK(CD"&"5"5f"===!#,1)(); :0 >
 &&(9: 	KKMLLN	
 KKMLLNsC   B$H'F- F)CF- F+F- +F- -;G((G+ +"HHc                      |j                  d      j                          fd} j                  j                  |       y )Nc                  p    	 j                   j                          y # t        j                  $ r Y y w xY wr)   )r   
put_nowaitr   	QueueFull)datar"   s   r$   _enqueuez,VoicePipeline._sd_callback.<locals>._enqueue   s4    &&11$7$$ s    55)reshapetobytesr<   rn   )r"   indatarx   _timestatusr   r   s   `     @r$   rh   zVoicePipeline._sd_callback   s3    ~~b!))+	 			&&x0r&   c                 r  K   t        j                  dj                  |      t         j                        j	                  t         j
                        dz  }| j                  4 d {    t        j                  | j                  j                  t        j                  |             d {   }d d d       d {    j                  j                         }|r<t        j!                  d|        | j"                  j%                  |       d {    y y 7 7 o7 a# 1 d {  7  sw Y   qxY w7 !w)Nr&   rG   rI   zTranscribed: )rL   rM   joinrN   rO   rP   r!   r   r1   r7   r   mxarraytextstripr/   r0   r   put)r"   rx   audioresultr   s        r$   rq   zVoicePipeline._process_audio   s     MM#((6*"((;BB2::NQXX 	 ===",,TXX->->PPF !={{  "KK-v./**..t444 	 !P !=== 5sh   A%D7'D(D7+AD ,D-D 1D7<D=AD7D5D7D D7 D2&D)'D2.D7c                    K   	 | j                   j                          d {   }| j                  |       d {    | j                   j                          V7 87 !wr)   )r   ri   _generate_response	task_done)r"   r   s     r$   r?   z!VoicePipeline._response_processor   sR     115577D))$///$$..0 7/s   AAAA AAc                 p  K   ddd}	 t         j                  d       dddd|dg}| j                  4 d {    t        j                  || j
                  | j                  |d       d {   }d d d       d {    t         j                  d	        |rNt        j                         | _        t        j                  | j                  || j                              | _        y y 7 7 7 q# 1 d {  7  sw Y   xY w# t        $ r"}t         j                  d
|        Y d }~y d }~ww xY ww)NFverbosec                d    |j                  |dd      }t        | |||      j                         S )NFT)tokenizeadd_generation_promptr   )apply_chat_templategenerate_textr   )r2   r3   messagesr   prompts        r$   _get_llm_responsez;VoicePipeline._generate_response.<locals>._get_llm_response   s:    225 3 F !iIOOQQr&   zGenerating response...systemzYou are a helpful voice assistant. You always respond with short sentences and never use punctuation like parentheses or colons that wouldn't appear in conversational speech.)rolecontentuserzGenerated response: zGeneration error: )r/   r0   r!   r   r1   r2   r3   Eventrl   r=   _speak_responserd   	Exceptionerror)r"   r   r   r   response_textes         r$   r   z VoicePipeline._generate_response   s    CH 	R	3KK01 %  P  D1H }}}&-&7&7%txxSX' ! %}
 KK.}o>?*1--/'(/(;(;((8O8OP)%  %! %}}}  	3LL-aS122	3s   D6/D C-D 2C3/C/0C34D ?C1 A+D +D6-D /C31D 3D9C<:DD 	D3D.)D6.D33D6r   cancel_eventc           	         K    j                   dt        j                  f fd}	  j                  4 d{    t        j                  | j
                  | j                   j                  |       d{    ddd      d{    y7 U7 7 	# 1 d{  7  sw Y   yxY w# t        j                  $ r Y yt        $ r }t        j                  d|       Y d}~yd}~ww xY ww)z
        Speak `text`, yielding PCM chunks into `self.output_audio_queue`.
        Playback can be interrupted at any moment by setting `cancel_event`.
        	cancel_evc                     | j                  ||dj                  d      D ]:  }|j                         r y j                  |j                  |j
                         < y )NTF)sample_raterw   r   r   )r   r   is_setrn   r   r   )r4   txtratequeuer   chunkr<   r"   s         r$   _tts_streamz2VoicePipeline._speak_response.<locals>._tts_stream   s[     #'#:#: &  ##%))%*:*:EKKHr&   NzSpeech synthesis error: %s)r<   r   r   r!   r1   r4   r   r   rr   r   r/   r   )r"   r   r   r   excr<   s   `    @r$   r   zVoicePipeline._speak_response   s     
 yy	I'-- 	I	<}}}''HH++++    %}} %}}} %% 	 	<LL5s;;	<s   $C*B* BB* <B9B:B>B* 	B
B* C*B* BB* B'BB'#B* &C*'B* *C'=C*?C'C"C*"C''C*c                 h  K   t        | j                        | _        	 	 | j                  j	                          d {   }| j                  j                  |       | j                  j                          X7 :# t        j                  t        f$ r | j                  j                           w xY ww)N)r   )r   r   ro   r   ri   queue_audior   r   rr   rs   rt   )r"   r   s     r$   r@   z%VoicePipeline._audio_output_processor  s     !d.E.EF	"5599;;''.''113 ; &&(9: 	KK	s    B2A: A8;A: :5B//B2N)
Q?      ?i>  i]        r   -mlx-community/whisper-large-v3-turbo-asr-fp16zQwen/Qwen2.5-0.5B-Instruct-4bitmlx-community/csm-1b-fp16)__name__
__module____qualname__r%   r8   rE   rX   r]   r>   rh   rq   r?   r   strr   r   r   r@    r&   r$   r   r      su      !A3-'@
;A&/.9v	1513D"<# "<W]] "<H
r&   r   c            	        K   t        j                  d      } | j                  dt        dd       | j                  dt        dd	       | j                  d
t        dd       | j                  dt        dd       | j                  dt
        dd       | j                  dt
        dd       | j                  dt        dd       | j                         }t        |j                  |j                  |j                  |j                  |j                  |j                  |j                        }|j                          d {    y 7 w)NzVoice Pipeline)descriptionz--stt_modelr   z	STT model)typedefaulthelpz--tts_modelr   z	TTS modelz--llm_modelz(mlx-community/Qwen2.5-0.5B-Instruct-4bitz	LLM modelz
--vad_moder   zVAD modez--silence_durationr   zSilence durationz--silence_thresholdr   zSilence thresholdz--streaming_intervalzStreaming interval)r   r   r   r#   r   r   r   )argparseArgumentParseradd_argumentr   re   float
parse_argsr   r   r   r   r#   r   r   r   rE   )parserargspipelines      r$   mainr     sM    $$1ABF
?	   C)D;   :	   3
K
5#<N   E4>Q   S!:N   D........0022H ..
s   EEE	E__main__)"r   r   loggingmlx.corecorer   numpyrL   sounddevicerf   r   mlx_lm.generater   r   mlx_lm.utilsr   r*   mlx_audio.stt.models.whisperr   r5   mlx_audio.tts.audio_playerr   mlx_audio.tts.utilsr   r.   basicConfigINFO	getLoggerr   r/   r   r   runrs   r   r&   r$   <module>r      s           5 ) 9 2 6   
,,U 
		8	$A AH&R zDF   s    B B! B!