Ë
    (täif…  ã                   óô  — d dl Z d dlmZ d dlZd dlmZ d dlZd dlm	Z
 d dlmZ ddlmZmZ ddlmZmZ ddlmZ  ej*                  e«      Zej0                  j2                   G d	„ d
e«      «       Zej0                  j2                   G d„ de«      «       Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ dej8                  «      Z  G d„ dej8                  «      Z! G d„ dej8                  «      Z" G d„ dej8                  «      Z# G d„ dej8                  «      Z$ G d„ dej8                  «      Z% G d„ d e&«      Z'e G d!„ d"ej8                  ee«      «       Z(y)#é    N)Úpartial)Ú
FrozenDicté   )ÚConfigMixinÚflax_register_to_config)Ú
BaseOutputÚloggingé   )ÚFlaxModelMixinc                   ó0   — e Zd ZU dZej
                  ed<   y)ÚFlaxDecoderOutputa;  
    Output of decoding method.

    Args:
        sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`):
            The decoded output sample from the last layer of the model.
        dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`):
            The `dtype` of the parameters.
    ÚsampleN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚjnpÚndarrayÚ__annotations__© ó    úh/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/models/vae_flax.pyr   r   "   s   … ñð �K‰KÔr   r   c                   ó   — e Zd ZU dZded<   y)ÚFlaxAutoencoderKLOutputaL  
    Output of AutoencoderKL encoding method.

    Args:
        latent_dist (`FlaxDiagonalGaussianDistribution`):
            Encoded outputs of `Encoder` represented as the mean and logvar of `FlaxDiagonalGaussianDistribution`.
            `FlaxDiagonalGaussianDistribution` allows for sampling latents from the distribution.
    Ú FlaxDiagonalGaussianDistributionÚlatent_distN)r   r   r   r   r   r   r   r   r   r   1   s   … ñð 4Ô3r   r   c                   ó^   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	d„ Z
y)ÚFlaxUpsample2DzÛ
    Flax implementation of 2D Upsample layer

    Args:
        in_channels (`int`):
            Input channels
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    Úin_channelsÚdtypec                 ó”   — t         j                  d«       t        j                  | j                  ddd| j
                  ¬«      | _        y )Nú”Flax classes are deprecated and will be removed in Diffusers v1.0.0. We recommend migrating to PyTorch classes or pinning your version of Diffusers.©é   r$   ©r
   r
   ©r%   r%   ©Úkernel_sizeÚstridesÚpaddingr    ©ÚloggerÚwarningÚnnÚConvr   r    Úconv©Úselfs    r   ÚsetupzFlaxUpsample2D.setupM   s?   € Ü�‰ð[ô	
ô —G‘GØ×ÑØØØ$Ø—*‘*ô
ˆ�	r   c                 ó¢   — |j                   \  }}}}t        j                  j                  |||dz  |dz  |fd¬«      }| j	                  |«      }|S )Nr   Únearest)ÚshapeÚmethod)r6   ÚjaxÚimageÚresizer0   )r2   Úhidden_statesÚbatchÚheightÚwidthÚchannelss         r   Ú__call__zFlaxUpsample2D.__call__Z   s_   € Ø)6×)<Ñ)<Ñ&ˆˆv�u˜hÜŸ	™	×(Ñ(ØØ˜& 1™* e¨a¡i°Ð:Øð )ó 
ˆð
 Ÿ	™	 -Ó0ˆØÐr   N©r   r   r   r   Úintr   r   Úfloat32r    r3   r@   r   r   r   r   r   ?   s+   … ñð ÓØ—{‘{€Eˆ3�9‰9Ó"ò
ór   r   c                   ó^   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	d„ Z
y)ÚFlaxDownsample2DzÝ
    Flax implementation of 2D Downsample layer

    Args:
        in_channels (`int`):
            Input channels
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r   r    c                 ó”   — t         j                  d«       t        j                  | j                  ddd| j
                  ¬«      | _        y )Nr"   r#   )r   r   ÚVALIDr'   r+   r1   s    r   r3   zFlaxDownsample2D.setups   s?   € Ü�‰ð[ô	
ô
 —G‘GØ×ÑØØØØ—*‘*ô
ˆ�	r   c                 óZ   — d}t        j                  ||¬«      }| j                  |«      }|S )N)©r   r   ©r   r
   rJ   rI   )Ú	pad_width)r   Úpadr0   )r2   r;   rL   s      r   r@   zFlaxDownsample2D.__call__�   s+   € Ø.ˆÜŸ™ ¸Ô=ˆØŸ	™	 -Ó0ˆØÐr   NrA   r   r   r   rE   rE   e   s+   … ñð ÓØ—{‘{€Eˆ3�9‰9Ó"ò
ór   rE   c                   ó˜   — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   ej                  Zej                  ed
<   d„ Zdd„Zy)ÚFlaxResnetBlock2Da  
    Flax implementation of 2D Resnet Block.

    Args:
        in_channels (`int`):
            Input channels
        out_channels (`int`):
            Output channels
        dropout (:obj:`float`, *optional*, defaults to 0.0):
            Dropout rate
        groups (:obj:`int`, *optional*, defaults to `32`):
            The number of groups to use for group norm.
        use_nin_shortcut (:obj:`bool`, *optional*, defaults to `None`):
            Whether to use `nin_shortcut`. This activates a new layer inside ResNet block
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r   NÚout_channelsç        Údropouté    ÚgroupsÚuse_nin_shortcutr    c                 ó®  — t         j                  d«       | j                  €| j                  n| j                  }t	        j
                  | j                  d¬«      | _        t	        j                  |ddd| j                  ¬«      | _
        t	        j
                  | j                  d¬«      | _        t	        j                  | j                  «      | _        t	        j                  |ddd| j                  ¬«      | _        | j                   €| j                  |k7  n| j                   }d | _        |r*t	        j                  |ddd| j                  ¬«      | _        y y )	Nr"   ç�íµ ÷Æ°>©Ú
num_groupsÚepsilonr#   r%   r&   r'   rG   )r,   r-   rO   r   r.   Ú	GroupNormrS   Únorm1r/   r    Úconv1Únorm2ÚDropoutrQ   Údropout_layerÚconv2rT   Úconv_shortcut)r2   rO   rT   s      r   r3   zFlaxResnetBlock2D.setup¢   s  € Ü�‰ð[ô	
ð
 ,0×+<Ñ+<Ð+D�t×'Ò'È$×J[ÑJ[ˆä—\‘\¨T¯[©[À$ÔGˆŒ
Ü—W‘WØØØØ$Ø—*‘*ô
ˆŒ
ô —\‘\¨T¯[©[À$ÔGˆŒ
ÜŸZ™Z¨¯©Ó5ˆÔÜ—W‘WØØØØ$Ø—*‘*ô
ˆŒ
ð @D×?TÑ?TÐ?\˜4×+Ñ+¨|Ò;Ðbf×bwÑbwÐà!ˆÔÙÜ!#§¡ØØ"ØØØ—j‘jô"ˆDÕð r   c                 óJ  — |}| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }t        j                  |«      }| j                  ||«      }| j                  |«      }| j                  �| j                  |«      }||z   S ©N)r[   r.   Úswishr\   r]   r_   r`   ra   )r2   r;   ÚdeterministicÚresiduals       r   r@   zFlaxResnetBlock2D.__call__É   s”   € Ø ˆØŸ
™
 =Ó1ˆÜŸ™ Ó/ˆØŸ
™
 =Ó1ˆàŸ
™
 =Ó1ˆÜŸ™ Ó/ˆØ×*Ñ*¨=¸-ÓHˆØŸ
™
 =Ó1ˆà×ÑÐ)Ø×)Ñ)¨(Ó3ˆHà˜xÑ'Ð'r   ©T)r   r   r   r   rB   r   rO   rQ   ÚfloatrS   rT   Úboolr   rC   r    r3   r@   r   r   r   rN   rN   ˆ   sU   … ñð$ ÓØ€L�#ÓØ€GˆUÓØ€FˆCÓØ!Ð�dÓ!Ø—{‘{€Eˆ3�9‰9Ó"ò%ôN(r   rN   c                   ó€   — e Zd ZU dZeed<   dZeed<   dZeed<   ej                  Z
ej                  ed<   d„ Zd	„ Zd
„ Zy)ÚFlaxAttentionBlockaì  
    Flax Convolutional based multi-head attention block for diffusion-based VAE.

    Parameters:
        channels (:obj:`int`):
            Input channels
        num_head_channels (:obj:`int`, *optional*, defaults to `None`):
            Number of attention heads
        num_groups (:obj:`int`, *optional*, defaults to `32`):
            The number of groups to use for group norm
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`

    r?   NÚnum_head_channelsrR   rX   r    c                 ó”  — t         j                  d«       | j                  �| j                  | j                  z  nd| _        t        t        j                  | j                  | j                  ¬«      }t        j                  | j                  d¬«      | _         |«        |«        |«       c| _        | _        | _         |«       | _        y )Nr"   r
   ©r    rV   rW   )r,   r-   rl   r?   Ú	num_headsr   r.   ÚDenser    rZ   rX   Ú
group_normÚqueryÚkeyÚvalueÚ	proj_attn)r2   Údenses     r   r3   zFlaxAttentionBlock.setupï   s‘   € Ü�‰ð[ô	
ð
 EI×DZÑDZÐDf˜Ÿ™¨$×*@Ñ*@Ò@ÐlmˆŒäœŸ™ $§-¡-°t·z±zÔBˆäŸ,™,°$·/±/È4ÔPˆŒÙ+0«7±E³G¹U»WÐ(ˆŒ
�D”H˜dœjÙ›ˆ�r   c                 ó�   — |j                   d d | j                  dfz   }|j                  |«      }t        j                  |d«      }|S )Néÿÿÿÿ©r   r   r
   r$   )r6   ro   Úreshaper   Ú	transpose)r2   Ú
projectionÚnew_projection_shapeÚnew_projections       r   Útranspose_for_scoresz'FlaxAttentionBlock.transpose_for_scoresý   sJ   € Ø)×/Ñ/°°Ð4¸¿¹ÈÐ7KÑKÐà#×+Ñ+Ð,@ÓAˆäŸ™ ~°|ÓDˆØÐr   c                 ó@  — |}|j                   \  }}}}| j                  |«      }|j                  |||z  |f«      }| j                  |«      }| j	                  |«      }| j                  |«      }	| j                  |«      }| j                  |«      }| j                  |	«      }	dt        j                  t        j                  | j                  | j                  z  «      «      z  }
t        j                  d||
z  ||
z  «      }t        j                  |d¬«      }t        j                  d|	|«      }t        j                  |d«      }|j                   d d | j                  fz   }|j                  |«      }| j!                  |«      }|j                  ||||f«      }||z   }|S )Nr
   z...qc,...kc->...qkrx   ©Úaxisz...kc,...qk->...qcry   éþÿÿÿ)r6   rq   rz   rr   rs   rt   r   ÚmathÚsqrtr?   ro   r   Úeinsumr.   Úsoftmaxr{   ru   )r2   r;   rf   r<   r=   r>   r?   rr   rs   rt   ÚscaleÚattn_weightsÚnew_hidden_states_shapes                r   r@   zFlaxAttentionBlock.__call__  s‚  € Ø ˆØ)6×)<Ñ)<Ñ&ˆˆv�u˜hàŸ™¨Ó6ˆà%×-Ñ-¨u°f¸u±nÀhÐ.OÓPˆà—
‘
˜=Ó)ˆØ�h‰h�}Ó%ˆØ—
‘
˜=Ó)ˆð ×)Ñ)¨%Ó0ˆØ×'Ñ'¨Ó,ˆØ×)Ñ)¨%Ó0ˆð ”D—I‘IœdŸi™i¨¯©¸¿¹Ñ(FÓGÓHÑHˆÜ—z‘zÐ"6¸À¹ÀsÈUÁ{ÓSˆÜ—z‘z ,°RÔ8ˆô Ÿ
™
Ð#7¸ÀÓMˆäŸ™ m°\ÓBˆØ"/×"5Ñ"5°c°rÐ":¸d¿m¹mÐ=MÑ"MÐØ%×-Ñ-Ð.EÓFˆàŸ™ }Ó5ˆØ%×-Ñ-¨u°f¸eÀXÐ.NÓOˆØ%¨Ñ0ˆØÐr   )r   r   r   r   rB   r   rl   rX   r   rC   r    r3   r   r@   r   r   r   rk   rk   Ú   sD   … ñð ƒMØ!Ð�sÓ!Ø€J�ÓØ—{‘{€Eˆ3�9‰9Ó"ò!òó r   rk   c                   ó¢   — e Zd ZU dZeed<   eed<   dZeed<   dZeed<   dZ	eed	<   d
Z
eed<   ej                  Zej                  ed<   d„ Zdd„Zy)ÚFlaxDownEncoderBlock2Daó  
    Flax Resnet blocks-based Encoder block for diffusion-based VAE.

    Parameters:
        in_channels (:obj:`int`):
            Input channels
        out_channels (:obj:`int`):
            Output channels
        dropout (:obj:`float`, *optional*, defaults to 0.0):
            Dropout rate
        num_layers (:obj:`int`, *optional*, defaults to 1):
            Number of Resnet layer block
        resnet_groups (:obj:`int`, *optional*, defaults to `32`):
            The number of groups to use for the Resnet block group norm
        add_downsample (:obj:`bool`, *optional*, defaults to `True`):
            Whether to add downsample layer
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r   rO   rP   rQ   r
   Ú
num_layersrR   Úresnet_groupsTÚadd_downsampler    c                 ó¦  — t         j                  d«       g }t        | j                  «      D ]h  }|dk(  r| j                  n| j
                  }t        || j
                  | j                  | j                  | j                  ¬«      }|j                  |«       Œj || _        | j                  r't        | j
                  | j                  ¬«      | _        y y ©Nr"   r   ©r   rO   rQ   rS   r    rn   )r,   r-   Úranger�   r   rO   rN   rQ   rŽ   r    ÚappendÚresnetsr�   rE   Údownsamplers_0©r2   r•   Úir   Ú	res_blocks        r   r3   zFlaxDownEncoderBlock2D.setupE  s²   € Ü�‰ð[ô	
ð
 ˆÜ�t—‘Ö'ˆAØ./°1ªf˜$×*Ò*¸$×:KÑ:KˆKä)Ø'Ø!×.Ñ.ØŸ™Ø×)Ñ)Ø—j‘jôˆIð �N‰N˜9Õ%ð (ð ˆŒà×ÒÜ"2°4×3DÑ3DÈDÏJÉJÔ"WˆDÕð r   c                 óv   — | j                   D ]  } |||¬«      }Œ | j                  r| j                  |«      }|S ©N©re   )r•   r�   r–   ©r2   r;   re   Úresnets       r   r@   zFlaxDownEncoderBlock2D.__call__\  s=   € Ø—l”lˆFÙ" =ÀÔN‰Mð #ð ×ÒØ ×/Ñ/°Ó>ˆMàÐr   Nrg   )r   r   r   r   rB   r   rQ   rh   r�   rŽ   r�   ri   r   rC   r    r3   r@   r   r   r   rŒ   rŒ   (  sZ   … ñð( ÓØÓØ€GˆUÓØ€J�ÓØ€M�3ÓØ€N�DÓØ—{‘{€Eˆ3�9‰9Ó"òXô.r   rŒ   c                   ó¢   — e Zd ZU dZeed<   eed<   dZeed<   dZeed<   dZ	eed	<   d
Z
eed<   ej                  Zej                  ed<   d„ Zdd„Zy)ÚFlaxUpDecoderBlock2Daï  
    Flax Resnet blocks-based Decoder block for diffusion-based VAE.

    Parameters:
        in_channels (:obj:`int`):
            Input channels
        out_channels (:obj:`int`):
            Output channels
        dropout (:obj:`float`, *optional*, defaults to 0.0):
            Dropout rate
        num_layers (:obj:`int`, *optional*, defaults to 1):
            Number of Resnet layer block
        resnet_groups (:obj:`int`, *optional*, defaults to `32`):
            The number of groups to use for the Resnet block group norm
        add_upsample (:obj:`bool`, *optional*, defaults to `True`):
            Whether to add upsample layer
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r   rO   rP   rQ   r
   r�   rR   rŽ   TÚadd_upsampler    c                 ó¦  — t         j                  d«       g }t        | j                  «      D ]h  }|dk(  r| j                  n| j
                  }t        || j
                  | j                  | j                  | j                  ¬«      }|j                  |«       Œj || _        | j                  r't        | j
                  | j                  ¬«      | _        y y r‘   )r,   r-   r“   r�   r   rO   rN   rQ   rŽ   r    r”   r•   r¡   r   Úupsamplers_0r—   s        r   r3   zFlaxUpDecoderBlock2D.setupƒ  s²   € Ü�‰ð[ô	
ð
 ˆÜ�t—‘Ö'ˆAØ./°1ªf˜$×*Ò*¸$×:KÑ:KˆKÜ)Ø'Ø!×.Ñ.ØŸ™Ø×)Ñ)Ø—j‘jôˆIð �N‰N˜9Õ%ð (ð ˆŒà×ÒÜ .¨t×/@Ñ/@ÈÏ
É
Ô SˆDÕð r   c                 óv   — | j                   D ]  } |||¬«      }Œ | j                  r| j                  |«      }|S r›   )r•   r¡   r£   r�   s       r   r@   zFlaxUpDecoderBlock2D.__call__š  s=   € Ø—l”lˆFÙ" =ÀÔN‰Mð #ð ×ÒØ ×-Ñ-¨mÓ<ˆMàÐr   Nrg   )r   r   r   r   rB   r   rQ   rh   r�   rŽ   r¡   ri   r   rC   r    r3   r@   r   r   r   r    r    f  sZ   … ñð( ÓØÓØ€GˆUÓØ€J�ÓØ€M�3ÓØ€L�$ÓØ—{‘{€Eˆ3�9‰9Ó"òTô.r   r    c                   ó˜   — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed	<   ej                  Zej                  ed
<   d„ Zdd„Zy)ÚFlaxUNetMidBlock2Da²  
    Flax Unet Mid-Block module.

    Parameters:
        in_channels (:obj:`int`):
            Input channels
        dropout (:obj:`float`, *optional*, defaults to 0.0):
            Dropout rate
        num_layers (:obj:`int`, *optional*, defaults to 1):
            Number of Resnet layer block
        resnet_groups (:obj:`int`, *optional*, defaults to `32`):
            The number of groups to use for the Resnet and Attention block group norm
        num_attention_heads (:obj:`int`, *optional*, defaults to `1`):
            Number of attention heads for each attention block
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r   rP   rQ   r
   r�   rR   rŽ   Únum_attention_headsr    c                 ód  — t         j                  d«       | j                  �| j                  nt        | j                  dz  d«      }t        | j                  | j                  | j                  || j                  ¬«      g}g }t        | j                  «      D ]‰  }t        | j                  | j                  || j                  ¬«      }|j                  |«       t        | j                  | j                  | j                  || j                  ¬«      }|j                  |«       Œ‹ || _        || _        y )Nr"   é   rR   r’   )r?   rl   rX   r    )r,   r-   rŽ   Úminr   rN   rQ   r    r“   r�   rk   r§   r”   r•   Ú
attentions)r2   rŽ   r•   r«   Ú_Ú
attn_blockr™   s          r   r3   zFlaxUNetMidBlock2D.setup¾  s  € Ü�‰ð[ô	
ð
 /3×.@Ñ.@Ð.L˜×*Ò*ÔRUÐVZ×VfÑVfÐjkÑVkÐmoÓRpˆô Ø ×,Ñ,Ø!×-Ñ-ØŸ™Ø$Ø—j‘jôð
ˆð ˆ
ä�t—‘Ö'ˆAÜ+Ø×)Ñ)Ø"&×":Ñ":Ø(Ø—j‘jô	ˆJð ×Ñ˜jÔ)ä)Ø ×,Ñ,Ø!×-Ñ-ØŸ™Ø$Ø—j‘jôˆIð �N‰N˜9Õ%ð! (ð$ ˆŒØ$ˆ�r   c                 ó®   —  | j                   d   ||¬«      }t        | j                  | j                   dd  «      D ]  \  }} ||«      } |||¬«      }Œ |S )Nr   rœ   r
   )r•   Úzipr«   )r2   r;   re   Úattnrž   s        r   r@   zFlaxUNetMidBlock2D.__call__è  s[   € Ø'˜Ÿ™ Q™¨À]ÔSˆÜ §¡°·±¸a¸bÐ1AÖB‰LˆD�&Ù  Ó/ˆMÙ" =ÀÔN‰Mð Cð Ðr   Nrg   )r   r   r   r   rB   r   rQ   rh   r�   rŽ   r§   r   rC   r    r3   r@   r   r   r   r¦   r¦   ¤  sU   … ñð$ ÓØ€GˆUÓØ€J�ÓØ€M�3ÓØ Ð˜Ó Ø—{‘{€Eˆ3�9‰9Ó"ò(%ôTr   r¦   c                   óà   — e Zd ZU dZdZeed<   dZeed<   dZe	e
df   ed<   dZe	edf   ed	<   d
Zeed<   dZeed<   dZe
ed<   dZeed<   ej$                  Zej&                  ed<   d„ Zddefd„Zy)ÚFlaxEncoderaô  
    Flax Implementation of VAE Encoder.

    This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
    subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
    general usage and behavior.

    Finally, this model supports inherent JAX features such as:
    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        in_channels (:obj:`int`, *optional*, defaults to 3):
            Input channels
        out_channels (:obj:`int`, *optional*, defaults to 3):
            Output channels
        down_block_types (:obj:`tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`):
            DownEncoder block type
        block_out_channels (:obj:`tuple[str]`, *optional*, defaults to `(64,)`):
            tuple[ containing the number of output channels for each block
        layers_per_block (:obj:`int`, *optional*, defaults to `2`):
            Number of Resnet layer for each block
        norm_num_groups (:obj:`int`, *optional*, defaults to `32`):
            norm num group
        act_fn (:obj:`str`, *optional*, defaults to `silu`):
            Activation function
        double_z (:obj:`bool`, *optional*, defaults to `False`):
            Whether to double the last output channels
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            Parameters `dtype`
    r$   r   rO   ©ÚDownEncoderBlock2D.Údown_block_types©é@   Úblock_out_channelsr   Úlayers_per_blockrR   Únorm_num_groupsÚsiluÚact_fnFÚdouble_zr    c           	      óè  — t         j                  d«       | j                  }t        j                  |d   ddd| j
                  ¬«      | _        g }|d   }t        | j                  «      D ]^  \  }}|}||   }|t        |«      dz
  k(  }t        ||| j                  | j                  | | j
                  ¬«      }|j                  |«       Œ` || _        t        |d	   | j                  d | j
                  ¬
«      | _        | j"                  rd| j$                  z  n| j$                  }	t        j&                  | j                  d¬«      | _        t        j                  |	ddd| j
                  ¬«      | _        y )Nr"   r   r#   r%   r&   r'   r
   )r   rO   r�   rŽ   r�   r    rx   ©r   rŽ   r§   r    r   rV   rW   )r,   r-   r¸   r.   r/   r    Úconv_inÚ	enumeraterµ   ÚlenrŒ   r¹   rº   r”   Údown_blocksr¦   Ú	mid_blockr½   rO   rZ   Úconv_norm_outÚconv_out)
r2   r¸   rÃ   Úoutput_channelr˜   r¬   Úinput_channelÚis_final_blockÚ
down_blockÚconv_out_channelss
             r   r3   zFlaxEncoder.setup  sh  € Ü�‰ð[ô	
ð
 "×4Ñ4Ðä—w‘wØ˜qÑ!ØØØ$Ø—*‘*ô
ˆŒð ˆØ+¨AÑ.ˆÜ˜d×3Ñ3Ö4‰DˆAˆqØ*ˆMØ/°Ñ2ˆNØ¤#Ð&8Ó"9¸AÑ"=Ñ=ˆNä/Ø)Ø+Ø×0Ñ0Ø"×2Ñ2Ø#1Ð1Ø—j‘jôˆJð ×Ñ˜zÕ*ð 5ð 'ˆÔô ,Ø*¨2Ñ.Ø×.Ñ.Ø $Ø—*‘*ô	
ˆŒð 6:·]²]˜A × 1Ñ 1Ò1È×HYÑHYÐÜŸ\™\°T×5IÑ5IÐSWÔXˆÔÜŸ™ØØØØ$Ø—*‘*ô
ˆ�r   re   c                 óò   — | j                  |«      }| j                  D ]  } |||¬«      }Œ | j                  ||¬«      }| j                  |«      }t	        j
                  |«      }| j                  |«      }|S r›   )rÀ   rÃ   rÄ   rÅ   r.   rd   rÆ   ©r2   r   re   Úblocks       r   r@   zFlaxEncoder.__call__T  su   € à—‘˜fÓ%ˆð ×%Ô%ˆEÙ˜6°Ô?‰Fð &ð —‘ °m�ÓDˆð ×#Ñ# FÓ+ˆÜ—‘˜&Ó!ˆØ—‘˜vÓ&ˆàˆr   Nrg   )r   r   r   r   r   rB   r   rO   rµ   ÚtupleÚstrr¸   r¹   rº   r¼   r½   ri   r   rC   r    r3   r@   r   r   r   r²   r²   ñ  s‘   … ñ ðD €K�ÓØ€L�#ÓØ(?Ð�e˜C ˜H‘oÓ?Ø*/Ð˜˜c 3˜h™Ó/ØÐ�cÓØ€O�SÓØ€FˆCÓØ€HˆdÓØ—{‘{€Eˆ3�9‰9Ó"ò4
ñl¨dô r   r²   c                   óÒ   — e Zd ZU dZdZeed<   dZeed<   dZe	e
df   ed<   dZe	edf   ed	<   d
Zeed<   dZeed<   dZe
ed<   ej                   Zej"                  ed<   d„ Zddefd„Zy)ÚFlaxDecoderaî  
    Flax Implementation of VAE Decoder.

    This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
    subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
    general usage and behavior.

    Finally, this model supports inherent JAX features such as:
    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        in_channels (:obj:`int`, *optional*, defaults to 3):
            Input channels
        out_channels (:obj:`int`, *optional*, defaults to 3):
            Output channels
        up_block_types (:obj:`tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`):
            UpDecoder block type
        block_out_channels (:obj:`tuple[str]`, *optional*, defaults to `(64,)`):
            tuple[ containing the number of output channels for each block
        layers_per_block (:obj:`int`, *optional*, defaults to `2`):
            Number of Resnet layer for each block
        norm_num_groups (:obj:`int`, *optional*, defaults to `32`):
            norm num group
        act_fn (:obj:`str`, *optional*, defaults to `silu`):
            Activation function
        double_z (:obj:`bool`, *optional*, defaults to `False`):
            Whether to double the last output channels
        dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32):
            parameters `dtype`
    r$   r   rO   ©ÚUpDecoderBlock2D.Úup_block_typesr¶   r¸   r   r¹   rR   rº   r»   r¼   r    c           	      óà  — t         j                  d«       | j                  }t        j                  |d   ddd| j
                  ¬«      | _        t        |d   | j                  d | j
                  ¬«      | _	        t        t        |«      «      }|d   }g }t        | j                  «      D ]c  \  }}|}||   }|t        |«      d	z
  k(  }t        ||| j                   d	z   | j                  | | j
                  ¬
«      }	|j#                  |	«       |}Œe || _        t        j&                  | j                  d¬«      | _        t        j                  | j*                  ddd| j
                  ¬«      | _        y )Nr"   rx   r#   r%   r&   r'   r¿   r   r
   )r   rO   r�   rŽ   r¡   r    rV   rW   )r,   r-   r¸   r.   r/   r    rÀ   r¦   rº   rÄ   ÚlistÚreversedrÁ   rÕ   rÂ   r    r¹   r”   Ú	up_blocksrZ   rÅ   rO   rÆ   )
r2   r¸   Úreversed_block_out_channelsrÇ   rÙ   r˜   r¬   Úprev_output_channelrÉ   Úup_blocks
             r   r3   zFlaxDecoder.setup“  sm  € Ü�‰ð[ô	
ð
 "×4Ñ4Ðô —w‘wØ˜rÑ"ØØØ$Ø—*‘*ô
ˆŒô ,Ø*¨2Ñ.Ø×.Ñ.Ø $Ø—*‘*ô	
ˆŒô '+¬8Ð4FÓ+GÓ&HÐ#Ø4°QÑ7ˆØˆ	Ü˜d×1Ñ1Ö2‰DˆAˆqØ"0ÐØ8¸Ñ;ˆNà¤#Ð&8Ó"9¸AÑ"=Ñ=ˆNä+Ø/Ø+Ø×0Ñ0°1Ñ4Ø"×2Ñ2Ø!/Ð/Ø—j‘jôˆHð ×Ñ˜XÔ&Ø"0Ñð 3ð" #ˆŒô  Ÿ\™\°T×5IÑ5IÐSWÔXˆÔÜŸ™Ø×ÑØØØ$Ø—*‘*ô
ˆ�r   re   c                 óò   — | j                  |«      }| j                  ||¬«      }| j                  D ]  } |||¬«      }Œ | j                  |«      }t	        j
                  |«      }| j                  |«      }|S r›   )rÀ   rÄ   rÙ   rÅ   r.   rd   rÆ   rÍ   s       r   r@   zFlaxDecoder.__call__Í  ss   € à—‘˜fÓ%ˆð —‘ °m�ÓDˆð —^”^ˆEÙ˜6°Ô?‰Fð $ð ×#Ñ# FÓ+ˆÜ—‘˜&Ó!ˆØ—‘˜vÓ&ˆàˆr   Nrg   )r   r   r   r   r   rB   r   rO   rÕ   rÏ   rÐ   r¸   r¹   rº   r¼   r   rC   r    r3   ri   r@   r   r   r   rÒ   rÒ   g  s†   … ñ ðD €K�ÓØ€L�#ÓØ&;€N�E˜#˜s˜(‘OÓ;Ø*/Ð˜˜c 3˜h™Ó/ØÐ�cÓØ€O�SÓØ€FˆCÓØ—{‘{€Eˆ3�9‰9Ó"ò8
ñt¨dô r   rÒ   c                   ó6   — e Zd Zdd„Zd„ Zd	d„Zg d¢fd„Zd„ Zy)
r   c                 ó®  — t        j                  |dd¬«      \  | _        | _        t        j                  | j                  dd«      | _        || _        t        j                  d| j                  z  «      | _        t        j                  | j                  «      | _        | j
                  r,t        j                  | j                  «      x| _        | _        y y )Nr   rx   r�   g      >Àg      4@ç      à?)
r   ÚsplitÚmeanÚlogvarÚclipre   ÚexpÚstdÚvarÚ
zeros_like)r2   Ú
parametersre   s      r   Ú__init__z)FlaxDiagonalGaussianDistribution.__init__à  s“   € ä!$§¡¨:°q¸rÔ!BÑˆŒ	�4”;Ü—h‘h˜tŸ{™{¨E°4Ó8ˆŒØ*ˆÔÜ—7‘7˜3 §¡Ñ,Ó-ˆŒÜ—7‘7˜4Ÿ;™;Ó'ˆŒØ×ÒÜ"%§.¡.°·±Ó";Ð;ˆDŒH�t•xð r   c                 óž   — | j                   | j                  t        j                  j	                  || j                   j
                  «      z  z   S rc   )râ   ræ   r8   ÚrandomÚnormalr6   )r2   rs   s     r   r   z'FlaxDiagonalGaussianDistribution.sampleê  s3   € Ø�y‰y˜4Ÿ8™8¤c§j¡j×&7Ñ&7¸¸T¿Y¹Y¿_¹_Ó&MÑMÑMÐMr   Nc                 óê  — | j                   rt        j                  dg«      S |€Fdt        j                  | j                  dz  | j
                  z   dz
  | j                  z
  g d¢¬«      z  S dt        j                  t        j                  | j                  |j                  z
  «      |j
                  z  | j
                  |j
                  z  z   dz
  | j                  z
  |j                  z   g d¢¬«      z  S )NrP   rà   r   g      ð?©r
   r   r$   r�   )re   r   ÚarrayÚsumrâ   rç   rã   Úsquare)r2   Úothers     r   Úklz#FlaxDiagonalGaussianDistribution.klí  sÄ   € Ø×ÒÜ—9‘9˜c˜UÓ#Ð#àˆ=ØœŸ™ §¡¨A¡°·±Ñ!8¸3Ñ!>ÀÇÁÑ!LÒS\Ô]Ñ]Ð]à”S—W‘WÜ�J‰J�t—y‘y 5§:¡:Ñ-Ó.°·±Ñ:¸T¿X¹XÈÏ	É	Ñ=QÑQÐTWÑWÐZ^×ZeÑZeÑeÐhm×htÑhtÑtÚô
ñ 
ð 	
r   rï   c                 ó@  — | j                   rt        j                  dg«      S t        j                  dt        j                  z  «      }dt        j
                  || j                  z   t        j                  || j                  z
  «      | j                  z  z   |¬«      z  S )NrP   g       @rà   r�   )
re   r   rð   ÚlogÚpirñ   rã   rò   râ   rç   )r2   r   r‚   Úlogtwopis       r   Únllz$FlaxDiagonalGaussianDistribution.nllù  su   € Ø×ÒÜ—9‘9˜c˜UÓ#Ð#ä—7‘7˜3¤§¡™<Ó(ˆØ”S—W‘W˜X¨¯©Ñ3´c·j±jÀÈ$Ï)É)ÑASÓ6TÐW[×W_ÑW_Ñ6_Ñ_ÐfjÔkÑkÐkr   c                 ó   — | j                   S rc   )râ   r1   s    r   Úmodez%FlaxDiagonalGaussianDistribution.mode   s   € Ø�y‰yÐr   )Frc   )r   r   r   rê   r   rô   rù   rû   r   r   r   r   r   ß  s"   „ ó<òNó

ò  )ó lór   r   c                   ó`  — e Zd ZU dZdZeed<   dZeed<   dZe	e
df   ed<   dZe	e
df   ed	<   d
Ze	edf   ed<   dZeed<   dZe
ed<   dZeed<   dZeed<   dZeed<   dZeed<   ej*                  Zej,                  ed<   d„ Zdej2                  defd„Zd"dedefd„Zd"dedefd„Zd#dedefd „Zy!)$ÚFlaxAutoencoderKLaµ  
    Flax implementation of a VAE model with KL loss for decoding latent representations.

    This model inherits from [`FlaxModelMixin`]. Check the superclass documentation for it's generic methods
    implemented for all models (such as downloading or saving).

    This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
    subclass. Use it as a regular Flax Linen module and refer to the Flax documentation for all matter related to its
    general usage and behavior.

    Inherent JAX features such as the following are supported:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        in_channels (`int`, *optional*, defaults to 3):
            Number of channels in the input image.
        out_channels (`int`, *optional*, defaults to 3):
            Number of channels in the output.
        down_block_types (`tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`):
            tuple[ of downsample block types.
        up_block_types (`tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`):
            tuple[ of upsample block types.
        block_out_channels (`tuple[str]`, *optional*, defaults to `(64,)`):
            tuple[ of block output channels.
        layers_per_block (`int`, *optional*, defaults to `2`):
            Number of ResNet layer for each block.
        act_fn (`str`, *optional*, defaults to `silu`):
            The activation function to use.
        latent_channels (`int`, *optional*, defaults to `4`):
            Number of channels in the latent space.
        norm_num_groups (`int`, *optional*, defaults to `32`):
            The number of groups for normalization.
        sample_size (`int`, *optional*, defaults to 32):
            Sample input size.
        scaling_factor (`float`, *optional*, defaults to 0.18215):
            The component-wise standard deviation of the trained latent space computed using the first batch of the
            training set. This is used to scale the latent space to have unit variance when training the diffusion
            model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the
            diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1
            / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image
            Synthesis with Latent Diffusion Models](https://huggingface.co/papers/2112.10752) paper.
        dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`):
            The `dtype` of the parameters.
    r$   r   rO   r³   .rµ   rÓ   rÕ   r¶   r¸   r
   r¹   r»   r¼   r©   Úlatent_channelsrR   rº   Úsample_sizegÿ²{ò°PÇ?Úscaling_factorr    c                 óâ  — t         j                  d«       t        | j                  j                  | j                  j
                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  d| j                  ¬«	      | _        t        | j                  j
                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  j                  | j                  ¬«      | _        t#        j$                  d| j                  j
                  z  ddd| j                  ¬«      | _        t#        j$                  | j                  j
                  ddd| j                  ¬«      | _        y )	Nr"   T)	r   rO   rµ   r¸   r¹   r¼   rº   r½   r    )r   rO   rÕ   r¸   r¹   rº   r¼   r    r   r%   rG   r'   )r,   r-   r²   Úconfigr   rþ   rµ   r¸   r¹   r¼   rº   r    ÚencoderrÒ   rO   rÕ   Údecoderr.   r/   Ú
quant_convÚpost_quant_convr1   s    r   r3   zFlaxAutoencoderKL.setupD  s[  € Ü�‰ð[ô	
ô
 #ØŸ™×/Ñ/ØŸ™×4Ñ4Ø!Ÿ[™[×9Ñ9Ø#Ÿ{™{×=Ñ=Ø!Ÿ[™[×9Ñ9Ø—;‘;×%Ñ%Ø ŸK™K×7Ñ7ØØ—*‘*ô

ˆŒô #ØŸ™×3Ñ3ØŸ™×1Ñ1ØŸ;™;×5Ñ5Ø#Ÿ{™{×=Ñ=Ø!Ÿ[™[×9Ñ9Ø ŸK™K×7Ñ7Ø—;‘;×%Ñ%Ø—*‘*ô	
ˆŒô Ÿ'™'Ø�—‘×+Ñ+Ñ+ØØØØ—*‘*ô
ˆŒô  "Ÿw™wØ�K‰K×'Ñ'ØØØØ—*‘*ô 
ˆÕr   ÚrngÚreturnc                 ó  — d| j                   | j                  | j                  f}t        j                  |t        j                  ¬«      }t
        j                  j                  |d«      \  }}}|||dœ}| j                  ||«      d   S )Nr
   rn   r$   )ÚparamsrQ   Úgaussianr
  )	r   rÿ   r   ÚzerosrC   r8   rì   rá   Úinit)r2   r  Úsample_shaper   Ú
params_rngÚdropout_rngÚgaussian_rngÚrngss           r   Úinit_weightszFlaxAutoencoderKL.init_weightsn  su   € à˜4×+Ñ+¨T×-=Ñ-=¸t×?OÑ?OÐPˆÜ—‘˜<¬s¯{©{Ô;ˆä03·
±
×0@Ñ0@ÀÀaÓ0HÑ-ˆ
�K Ø$°È,ÑWˆà�y‰y˜˜vÓ& xÑ0Ð0r   re   Úreturn_dictc                 ó®   — t        j                  |d«      }| j                  ||¬«      }| j                  |«      }t	        |«      }|s|fS t        |¬«      S )N©r   r   r$   r
   rœ   )r   )r   r{   r  r  r   r   )r2   r   re   r  r;   ÚmomentsÚ	posteriors          r   ÚencodezFlaxAutoencoderKL.encodex  sR   € Ü—‘˜v |Ó4ˆàŸ™ V¸=˜ÓIˆØ—/‘/ -Ó0ˆÜ4°WÓ=ˆ	áØ�<Ðä&°9Ô=Ð=r   c                 ó  — |j                   d   | j                  j                  k7  rt        j                  |d«      }| j                  |«      }| j                  ||¬«      }t        j                  |d«      }|s|fS t        |¬«      S )Nrx   r  rœ   )r   r$   r
   r   ©r   )r6   r  rþ   r   r{   r  r  r   )r2   Úlatentsre   r  r;   s        r   ÚdecodezFlaxAutoencoderKL.decode„  sv   € Ø�=‰=˜Ñ §¡× ;Ñ ;Ò;Ü—m‘m G¨\Ó:ˆGà×,Ñ,¨WÓ5ˆØŸ™ ]À-˜ÓPˆäŸ™ m°\ÓBˆáØ!Ð#Ð#ä ¨Ô6Ð6r   c                 ó  — | j                  |||¬«      }|r-| j                  d«      }|j                  j                  |«      }n|j                  j	                  «       }| j                  ||¬«      j                  }|s|fS t        |¬«      S )N)re   r  r  )r  r  )r  Úmake_rngr   r   rû   r  r   )r2   r   Úsample_posteriorre   r  r  r  r;   s           r   r@   zFlaxAutoencoderKL.__call__’  s€   € Ø—K‘K °mÐQ\�KÓ]ˆ	ÙØ—-‘- 
Ó+ˆCØ%×1Ñ1×8Ñ8¸Ó=‰Mà%×1Ñ1×6Ñ6Ó8ˆMà—‘˜]¸�ÓD×KÑKˆáØ�9Ðä ¨Ô/Ð/r   N)TT)FTT) r   r   r   r   r   rB   r   rO   rµ   rÏ   rÐ   rÕ   r¸   r¹   r¼   rþ   rº   rÿ   r   rh   r   rC   r    r3   r8   ÚArrayr   r  ri   r  r  r@   r   r   r   rý   rý     sü   … ñ/ðb €K�ÓØ€L�#ÓØ(?Ð�e˜C ˜H‘oÓ?Ø&;€N�E˜#˜s˜(‘OÓ;Ø*/Ð˜˜c 3˜h™Ó/ØÐ�cÓØ€FˆCÓØ€O�SÓØ€O�SÓØ€K�ÓØ#€N�EÓ#Ø—{‘{€Eˆ3�9‰9Ó"ò(
ðT1 §	¡	ð 1¨jó 1ñ
>¨Dð 
>Àdó 
>ñ7¨Tð 7Àtó 7ñ0Àdð 0Ð`dô 0r   rý   ))r„   Ú	functoolsr   ÚflaxÚ
flax.linenÚlinenr.   r8   Ú	jax.numpyÚnumpyr   Úflax.core.frozen_dictr   Úconfiguration_utilsr   r   Úutilsr   r	   Úmodeling_flax_utilsr   Ú
get_loggerr   r,   ÚstructÚ	dataclassr   r   ÚModuler   rE   rN   rk   rŒ   r    r¦   r²   rÒ   Úobjectr   rý   r   r   r   Ú<module>r1     sY  ðó" Ý ã Ý Û 
Ý Ý ,ç Fß 'Ý /ð 
ˆ×	Ñ	˜HÓ	%€ð ‡�×Ñô˜
ó ó ðð ‡�×Ñô
4˜jó 
4ó ð
4ô#�R—Y‘Yô #ôL �r—y‘yô  ôFO(˜Ÿ	™	ô O(ôdK˜Ÿ™ô Kô\;˜RŸY™Yô ;ô|;˜2Ÿ9™9ô ;ô|J˜Ÿ™ô JôZs�"—)‘)ô sôlu�"—)‘)ô uôp" vô "ðJ ôZ0˜Ÿ	™	 >°;ó Z0ó ñZ0r   