
      i                         d Z ddlmZmZ ddlmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ  G d
 dej,                        Z G d deee            Z G d de      Zy)zDemo of a simple LSTM language model.

Code is adapted from the PyTorch examples at
https://github.com/pytorch/examples/blob/main/word_language_model

    )IteratorSized)OptionalN)Tensor)	Optimizer)
DataLoaderSampler)LightningModule	WikiText2c                        e Zd Z	 ddededededef
 fdZddZd	ed
eeef   deeef   fdZ	dedeeef   fdZ
 xZS )
SimpleLSTM
vocab_sizeninpnhidnlayersdropoutc                 J   t         |           || _        t        j                  |      | _        t        j                  ||      | _        t        j                  ||||d      | _	        t        j                  ||      | _        || _        || _        | j                          y )NT)r   batch_first)super__init__r   nnDropoutdrop	EmbeddingencoderLSTMrnnLineardecoderr   r   init_weights)selfr   r   r   r   r   	__class__s         q/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/lightning/pytorch/demos/lstm.pyr   zSimpleLSTM.__init__   s}     	$JJw'	||J5774wTRyyz2	    returnc                 >   t         j                  j                  | j                  j                  dd       t         j                  j                  | j                  j                         t         j                  j                  | j                  j                  dd       y )Ngg?)r   inituniform_r   weightzeros_r    biasr"   s    r$   r!   zSimpleLSTM.init_weights$   s\    
,,dC8
t||(()
,,dC8r%   inputhiddenc                    | j                  | j                  |            }| j                  ||      \  }}| j                  |      }| j                  |      j	                  d| j
                        }t        j                  |d      |fS )N   )dim)r   r   r   r    viewr   Flog_softmax)r"   r.   r/   emboutputdecodeds         r$   forwardzSimpleLSTM.forward)   sp    iiU+,#v.6",,v&++B@}}W!,f44r%   
batch_sizec                     t        | j                               }|j                  | j                  || j                        |j                  | j                  || j                        fS N)next
parameters	new_zerosr   r   )r"   r;   r*   s      r$   init_hiddenzSimpleLSTM.init_hidden0   sQ    doo'(T\\:tyyAT\\:tyyA
 	
r%   )     rC      g?r&   N)__name__
__module____qualname__intfloatr   r!   r   tupler:   rA   __classcell__r#   s   @r$   r   r      s    lo-0>ARUdi9
5V 5U66>-B 5uVU[^G\ 5
c 
eFFN.C 
r%   r   c                   P     e Zd Zdededdf fdZdeee      fdZdefdZ	 xZ
S )SequenceSamplerdatasetr;   r&   Nc                     t         |           || _        || _        t	        | j                        | j                  z  | _        y r=   )r   r   rP   r;   len
chunk_size)r"   rP   r;   r#   s      r$   r   zSequenceSampler.__init__9   s5    $dll+t>r%   c           	   #      K   t        | j                        }t        | j                        D ]4  }t	        t        |||| j
                  z  z
  | j                               6 y wr=   )rR   rP   rangerS   listr;   )r"   nis      r$   __iter__zSequenceSampler.__iter__?   sM     t'AuQQ%8 94??KLL (s   A#A%c                     | j                   S r=   )rS   r-   s    r$   __len__zSequenceSampler.__len__D   s    r%   )rF   rG   rH   r   rI   r   r   rV   rY   r[   rL   rM   s   @r$   rO   rO   8   s@    ? ?3 ?4 ?M(49- M
 r%   rO   c                   l     e Zd Zddef fdZddZdeeef   dedefdZddZ	de
fd	Zdefd
Z xZS )LightningLSTMr   c                 R    t         |           t        |      | _        d | _        y )N)r   )r   r   r   modelr/   )r"   r   r#   s     r$   r   zLightningLSTM.__init__I   s!    :6
7;r%   r&   c                     d | _         y r=   )r/   r-   s    r$   on_train_epoch_endz LightningLSTM.on_train_epoch_endN   s	    r%   batch	batch_idxc                    |\  }}| j                   /| j                  j                  |j                  d            | _         | j                   d   j	                         | j                   d   j	                         f| _         | j                  || j                         \  }| _         t        j                  ||j                  d            }| j                  d|d       |S )Nr   r2   r1   
train_lossT)prog_bar)	r/   r_   rA   sizedetachr5   nll_lossr4   log)r"   rb   rc   r.   targetr8   losss          r$   training_stepzLightningLSTM.training_stepQ   s    v;;**00A?DK{{1~,,.A0E0E0GH"jj<zz&&++b/2td3r%   c                     t        d       y )NT)downloadr   r-   s    r$   prepare_datazLightningLSTM.prepare_data[   s
    4 r%   c                 F    t               }t        |t        |d            S )N   )r;   )batch_sampler)r   r   rO   )r"   rP   s     r$   train_dataloaderzLightningLSTM.train_dataloader^   s    +'UW1XYYr%   c                 `    t         j                  j                  | j                         d      S )Ng      4@)lr)torchoptimSGDr?   r-   s    r$   configure_optimizersz"LightningLSTM.configure_optimizersb   s     {{t0T::r%   )rB   rE   )rF   rG   rH   rI   r   ra   rK   r   rm   rp   r   rt   r   rz   rL   rM   s   @r$   r]   r]   H   sV    <3 <
5#8 S V !Z* Z;i ;r%   r]   )__doc__collections.abcr   r   typingr   rw   torch.nnr   torch.nn.functional
functionalr5   r   torch.optimr   torch.utils.datar   r	   lightning.pytorch.corer
   #lightning.pytorch.demos.transformerr   Moduler   rV   rI   rO   r]    r%   r$   <module>r      s_    ,       ! 0 2 9
 
Dgd3i(  ;O ;r%   