U
    ,-eY                     @   s  d Z ddlmZ ddlmZmZmZmZ ddlZddl	Zddlm
Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZmZ ddlmZmZmZ eeZdZdgZ dDej!ej"ee# dddZ$ej!ej!dddZ%ej!ej!dddZ&eG dd deZ'eG dd deZ(eG dd deZ)G dd de
j*Z+G dd  d e
j*Z,G d!d" d"e
j*Z-G d#d$ d$e
j*Z.G d%d& d&e
j*Z/G d'd( d(eZ0d)Z1d*Z2d+Z3d,Z4G d-d. d.e
j*Z5dEej6ej"ej7e#d/d0d1Z8G d2d3 d3e
j*Z9ed4e1G d5d6 d6e0Z:G d7d8 d8e
j*Z;ed9e1G d:d; d;e0Z<ee1G d<d= d=e0Z=ed>e1G d?d@ d@e0Z>edAe1G dBdC dCe0Z?dS )Fz PyTorch CLIP model.    )	dataclass)AnyOptionalTupleUnionN)nn   )ACT2FN)BaseModelOutputBaseModelOutputWithPooling)PreTrainedModel)ModelOutputadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )
CLIPConfigCLIPTextConfigCLIPVisionConfigzopenai/clip-vit-base-patch32)maskdtypetgt_lenc                 C   sj   |   \}}|dk	r|n|}| ddddddf |d|||}d| }||tjt|jS )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    Nr         ?)sizeexpandtoZmasked_filltorchboolfinfomin)r   r   r   bszsrc_lenZexpanded_maskZinverted_mask r#   g/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py_expand_mask1   s
    *r%   )logitsreturnc                 C   s   t j| tjt| | jdS )Ndevice)r   
functionalZcross_entropyr   arangelenr)   )r&   r#   r#   r$   contrastive_lossA   s    r-   )
similarityr'   c                 C   s    t | }t |  }|| d S )Ng       @)r-   t)r.   Zcaption_lossZ
image_lossr#   r#   r$   	clip_lossE   s    r0   c                   @   s^   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dS )CLIPVisionModelOutputa  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nimage_embedslast_hidden_statehidden_states
attentions)__name__
__module____qualname____doc__r2   r   r   FloatTensor__annotations__r3   r4   r   r5   r#   r#   r#   r$   r1   K   s
   
r1   c                   @   s^   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dS )CLIPTextModelOutputa  
    Base class for text model's outputs that also contains a pooling of the last hidden states.

    Args:
        text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The text embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Ntext_embedsr3   r4   r5   )r6   r7   r8   r9   r=   r   r   r:   r;   r3   r4   r   r5   r#   r#   r#   r$   r<   h   s
   
r<   c                   @   s   e Zd ZU dZdZeej ed< dZ	ejed< dZ
ejed< dZejed< dZejed< dZeed< dZeed	< ee d
ddZdS )
CLIPOutputa  
    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
            Contrastive loss for image-text similarity.
        logits_per_image:(`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
            The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
            similarity scores.
        logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
            The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
            similarity scores.
        text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
            The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`].
        image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
            The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`].
        text_model_output(`BaseModelOutputWithPooling`):
            The output of the [`CLIPTextModel`].
        vision_model_output(`BaseModelOutputWithPooling`):
            The output of the [`CLIPVisionModel`].
    Nlosslogits_per_imagelogits_per_textr=   r2   text_model_outputvision_model_outputr'   c                    s   t  fdd  D S )Nc                 3   s,   | ]$}|d kr | nt  | V  qdS ))rB   rC   N)getattrto_tuple).0kselfr#   r$   	<genexpr>   s   z&CLIPOutput.to_tuple.<locals>.<genexpr>)tuplekeysrI   r#   rI   r$   rF      s    zCLIPOutput.to_tuple)r6   r7   r8   r9   r?   r   r   r:   r;   r@   rA   r=   r2   rB   r   rC   r   r   rF   r#   r#   r#   r$   r>      s   
r>   c                       s6   e Zd Zed fddZejejdddZ  Z	S )CLIPVisionEmbeddingsconfigc                    s   t    || _|j| _|j| _|j| _tt	
| j| _tj|j| j| j| jdd| _| j| j d | _| jd | _t| j| j| _| jdt	| jddd d S )NF)Zin_channelsZout_channelsZkernel_sizeZstridebias   r   position_idsr   
persistent)super__init__rP   hidden_size	embed_dimZ
image_sizeZ
patch_sizer   	Parameterr   Zrandnclass_embeddingZConv2dZnum_channelspatch_embeddingZnum_patchesZnum_positions	Embeddingposition_embeddingregister_bufferr+   r   rJ   rP   	__class__r#   r$   rY      s"    
zCLIPVisionEmbeddings.__init__)pixel_valuesr'   c                 C   sn   |j d }| jjj}| |j|d}|ddd}| j|dd}t	j
||gdd}|| | j }|S )Nr   )r   rR   r   rU   dim)shaper^   weightr   r   flatten	transposer]   r   r   catr`   rS   )rJ   re   Z
batch_sizeZtarget_dtypeZpatch_embedsZclass_embeds
embeddingsr#   r#   r$   forward   s    

zCLIPVisionEmbeddings.forward)
r6   r7   r8   r   rY   r   r:   Tensorrn   __classcell__r#   r#   rc   r$   rN      s   rN   c                       sL   e Zd Zed fddZdeej eej eej ej	dddZ
  ZS )	CLIPTextEmbeddingsrO   c                    sR   t    |j}t|j|| _t|j|| _| j	dt
|jddd d S )NrS   rT   FrV   )rX   rY   rZ   r   r_   Z
vocab_sizetoken_embeddingZmax_position_embeddingsr`   ra   r   r+   r   rJ   rP   r[   rc   r#   r$   rY      s    
  zCLIPTextEmbeddings.__init__N)	input_idsrS   inputs_embedsr'   c                 C   sb   |d k	r|j d n|j d }|d kr:| jd d d |f }|d krL| |}| |}|| }|S )NrU   )rh   rS   rr   r`   )rJ   rt   rS   ru   Z
seq_lengthZposition_embeddingsrm   r#   r#   r$   rn      s    

zCLIPTextEmbeddings.forward)NNN)r6   r7   r8   r   rY   r   r   
LongTensorr:   ro   rn   rp   r#   r#   rc   r$   rq      s      rq   c                       sz   e Zd ZdZ fddZejeedddZdeje	ej e	ej e	e
 eeje	ej e	eej  f d	d
dZ  ZS )CLIPAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                    s   t    || _|j| _|j| _| j| j | _| j| j | jkrZtd| j d| j d| jd | _	|j
| _t| j| j| _t| j| j| _t| j| j| _t| j| j| _d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).      )rX   rY   rP   rZ   r[   Znum_attention_heads	num_headshead_dim
ValueErrorscaleZattention_dropoutdropoutr   Lineark_projv_projq_projout_projrb   rc   r#   r$   rY      s    
zCLIPAttention.__init__)tensorseq_lenr!   c                 C   s    | ||| j| jdd S )Nr   rR   )viewrz   r{   rk   
contiguous)rJ   r   r   r!   r#   r#   r$   _shape  s    zCLIPAttention._shapeNFr4   attention_maskcausal_attention_maskoutput_attentionsr'   c                 C   s  |  \}}}| || j }| | |d|}	| | |d|}
|| j d| jf}| |||j| }|	j| }	|
j| }
|	 d}t	
||	dd}|  || j ||fkrtd|| j ||f d|   |dk	rD|  |d||fkrtd|d||f d|   ||| j||| }||| j ||}|dk	r|  |d||fkrtd|d||f d|   ||| j||| }||| j ||}tjj|dd}|r||| j||}||| j ||}nd}tjj|| j| jd	}t	
||
}|  || j || jfkrRtd
|| j|| jf d|   ||| j|| j}|dd}||||}| |}||fS )z#Input shape: Batch x Time x ChannelrU   r   rR   z$Attention weights should be of size z	, but is Nz!Attention mask should be of size rf   )ptrainingz `attn_output` should be of size )r   r   r}   r   r   r   rz   r{   r   r   Zbmmrk   r|   r   r*   Zsoftmaxr~   r   Zreshaper   )rJ   r4   r   r   r   r!   r   r[   Zquery_statesZ
key_statesZvalue_statesZ
proj_shaper"   attn_weightsZattn_weights_reshapedZ
attn_probsZattn_outputr#   r#   r$   rn     sX    	





zCLIPAttention.forward)NNF)r6   r7   r8   r9   rY   r   ro   intr   r   r   r   rn   rp   r#   r#   rc   r$   rx      s      rx   c                       s0   e Zd Z fddZejejdddZ  ZS )CLIPMLPc                    sD   t    || _t|j | _t|j|j	| _
t|j	|j| _d S N)rX   rY   rP   r	   Z
hidden_actactivation_fnr   r   rZ   Zintermediate_sizefc1fc2rb   rc   r#   r$   rY   W  s
    
zCLIPMLP.__init__)r4   r'   c                 C   s"   |  |}| |}| |}|S r   )r   r   r   )rJ   r4   r#   r#   r$   rn   ^  s    


zCLIPMLP.forward)r6   r7   r8   rY   r   ro   rn   rp   r#   r#   rc   r$   r   V  s   r   c                       sJ   e Zd Zed fddZdejejejee e	ej
 dddZ  ZS )	CLIPEncoderLayerrO   c                    sR   t    |j| _t|| _tj| j|jd| _	t
|| _tj| j|jd| _d S N)Zeps)rX   rY   rZ   r[   rx   	self_attnr   	LayerNormlayer_norm_epslayer_norm1r   mlplayer_norm2rb   rc   r#   r$   rY   f  s    


zCLIPEncoderLayer.__init__Fr   c                 C   sd   |}|  |}| j||||d\}}|| }|}| |}| |}|| }|f}|r`||f7 }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r4   r   r   r   )r   r   r   r   )rJ   r4   r   r   r   Zresidualr   outputsr#   r#   r$   rn   n  s"    




zCLIPEncoderLayer.forward)F)r6   r7   r8   r   rY   r   ro   r   r   r   r:   rn   rp   r#   r#   rc   r$   r   e  s    r   c                   @   s.   e Zd ZdZeZdZdZdd Zd
ddZ	d	S )CLIPPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ZclipTc                 C   s  | j j}t|trF|jjjjd|d d |jjjjd|d d n&t|t	r| j j}t
jj|jd|jd | d t
jj|jj|j j| d t
jj|jj|j j| d nt|trF| j j}|jd d|j j d  | }|jd | }t
jj|jj|d t
jj|jj|d t
jj|jj|d t
jj|jj|d n&t|tr| j j}|j jd d|j j d  | }d|j j d | }t
jj|jj|d t
jj|jj|d nt|tr
t
jj|jj|jd | j j d t
jj|jj|jd | j j d nbt|tr<t
jj|jj| j jd | j j d n0t|t rlt
jj|jj| j jd | j j d t|t
j!r|j"j#  |jj$d t|t
j%r|j"dk	r|j"j#  dS )	zInitialize the weightsg        g{Gz?)Zmeanstdry   )r   rR   r   N)&rP   Zinitializer_factor
isinstancerq   rr   ri   dataZnormal_r`   rN   r   initr]   r[   r^   Zinitializer_rangerx   num_hidden_layersr   r   r   r   r   rZ   r   r   	CLIPModeltext_projectiontext_embed_dimvisual_projectionvision_embed_dimCLIPVisionModelWithProjectionCLIPTextModelWithProjectionr   rQ   Zzero_Zfill_r   )rJ   modulefactorZin_proj_stdZout_proj_stdZfc_stdr#   r#   r$   _init_weights  s`    

 z!CLIPPreTrainedModel._init_weightsFc                 C   s   t |tr||_d S r   )r   CLIPEncodergradient_checkpointing)rJ   r   valuer#   r#   r$   _set_gradient_checkpointing  s    
z/CLIPPreTrainedModel._set_gradient_checkpointingN)F)
r6   r7   r8   r9   r   config_classZbase_model_prefixZsupports_gradient_checkpointingr   r   r#   r#   r#   r$   r     s   5r   a=  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
aE  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                	       s`   e Zd ZdZed fddZd	eej eej ee	 ee	 ee	 e
eef dddZ  ZS )
r   z
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`CLIPEncoderLayer`].

    Args:
        config: CLIPConfig
    rO   c                    s:   t     | _t fddt jD | _d| _d S )Nc                    s   g | ]}t  qS r#   )r   )rG   _rO   r#   r$   
<listcomp>K  s     z(CLIPEncoder.__init__.<locals>.<listcomp>F)	rX   rY   rP   r   Z
ModuleListranger   layersr   rb   rc   rO   r$   rY   H  s    
 zCLIPEncoder.__init__N)r   r   r   output_hidden_statesreturn_dictr'   c                    s   dk	r n| j j |dk	r |n| j j}|dk	r4|n| j j}|rDdnd} rPdnd}|}	t| jD ]r\}
}|rx||	f }| jr| jr fdd}tj	j

|||	||}n||	|| d}|d }	 rb||d f }qb|r||	f }|stdd	 |	||fD S t|	||d
S )a  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr#   c                    s    fdd}|S )Nc                     s    | f S r   r#   )inputs)r   r   r#   r$   custom_forward  s    zJCLIPEncoder.forward.<locals>.create_custom_forward.<locals>.custom_forwardr#   )r   r   r   )r   r$   create_custom_forward  s    z2CLIPEncoder.forward.<locals>.create_custom_forwardr   r   r   c                 s   s   | ]}|d k	r|V  qd S r   r#   )rG   vr#   r#   r$   rK     s      z&CLIPEncoder.forward.<locals>.<genexpr>)r3   r4   r5   )rP   r   r   use_return_dict	enumerater   r   r   r   utils
checkpointrL   r
   )rJ   ru   r   r   r   r   r   Zencoder_statesZall_attentionsr4   idxZencoder_layerr   Zlayer_outputsr#   r   r$   rn   N  sH    &

  zCLIPEncoder.forward)NNNNN)r6   r7   r8   r9   r   rY   r   r   ro   r   r   r   r
   rn   rp   r#   r#   rc   r$   r   ?  s   	     
r   )input_ids_shaper   r)   past_key_values_lengthc                 C   s   | \}}t j||ft |j|d}t j|d|d}|||d |ddk d ||}|dkrt j	t j
||||d|gdd}|ddddddf |d||| S )zB
    Make causal mask used for bi-directional self-attention.
    r(   rU   r   r   r   r)   rf   N)r   fullr   r    r+   r   Zmasked_fill_r   r   rl   Zzerosr   )r   r   r)   r   r!   r   r   Z	mask_condr#   r#   r$   _make_causal_mask  s    "
 r   c                       sx   e Zd Zed fddZeeeeedd	e	e
j e	e
j e	e
j e	e e	e e	e eeef dddZ  ZS )
CLIPTextTransformerrO   c                    sH   t    || _|j}t|| _t|| _tj	||j
d| _|j| _d S r   )rX   rY   rP   rZ   rq   rm   r   encoderr   r   r   final_layer_normeos_token_idrs   rc   r#   r$   rY     s    


zCLIPTextTransformer.__init__output_typer   Nrt   r   rS   r   r   r   r'   c                 C   sn  |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}|dkrLtd| }|d|d }| j||d}t||j	|j
d}	|dk	rt||j	}| j|||	|||d}
|
d }| |}| jdkr|tj|jd |j
d|jtj|j
d	jdd
f }n>|tj|jd |j
d|jtj|j
d	| jk jdd
f }|sZ||f|
dd  S t|||
j|
jdS )
        Returns:

        NzYou have to specify input_idsrU   )rt   rS   r(   )ru   r   r   r   r   r   r   rR   r   rf   r   r3   Zpooler_outputr4   r5   )rP   r   r   r   r|   r   r   rm   r   r   r)   r%   r   r   r   r   r+   rh   r   r   Zargmaxr   r4   r5   )rJ   rt   r   rS   r   r   r   Zinput_shaper4   r   encoder_outputsr3   pooled_outputr#   r#   r$   rn     sV    	
zCLIPTextTransformer.forward)NNNNNN)r6   r7   r8   r   rY   r   CLIP_TEXT_INPUTS_DOCSTRINGr   r   r   r   ro   r   r   r   rn   rp   r#   r#   rc   r$   r     s$   
      
r   z?The text model from CLIP without any head or projection on top.c                       s   e Zd ZeZddgZed fddZejdddZ	d	d
 Z
eeeeeddeej eej eej ee ee ee eeef dddZ  ZS )CLIPTextModelrq   r   rO   c                    s"   t  | t|| _|   d S r   )rX   rY   r   
text_model	post_initrb   rc   r#   r$   rY     s    
zCLIPTextModel.__init__rD   c                 C   s
   | j jjS r   r   rm   rr   rI   r#   r#   r$   get_input_embeddings$  s    z"CLIPTextModel.get_input_embeddingsc                 C   s   || j j_d S r   r   rJ   r   r#   r#   r$   set_input_embeddings'  s    z"CLIPTextModel.set_input_embeddingsr   Nr   c                 C   s*   |dk	r|n| j j}| j||||||dS )aK  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, CLIPTextModel

        >>> model = CLIPTextModel.from_pretrained("openai/clip-vit-base-patch32")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```Nrt   r   rS   r   r   r   )rP   r   r   )rJ   rt   r   rS   r   r   r   r#   r#   r$   rn   *  s    zCLIPTextModel.forward)NNNNNN)r6   r7   r8   r   r   _no_split_modulesrY   r   Moduler   r   r   r   r   r   r   r   ro   r   r   r   rn   rp   r#   r#   rc   r$   r     s,   
      
r   c                
       sh   e Zd Zed fddZeeeeedd	e	e
j e	e e	e e	e eeef dddZ  ZS )
CLIPVisionTransformerrO   c                    sR   t    || _|j}t|| _tj||jd| _	t
|| _tj||jd| _d S r   )rX   rY   rP   rZ   rN   rm   r   r   r   pre_layrnormr   r   post_layernormrs   rc   r#   r$   rY   S  s    


zCLIPVisionTransformer.__init__r   Nre   r   r   r   r'   c           	      C   s   |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}|dkrLtd| |}| |}| j||||d}|d }|dddddf }| |}|s||f|dd  S t	|||j
|jdS )r   Nz You have to specify pixel_values)ru   r   r   r   r   r   r   )rP   r   r   r   r|   rm   r   r   r   r   r4   r5   )	rJ   re   r   r   r   r4   r   r3   r   r#   r#   r$   rn   ]  s2    


zCLIPVisionTransformer.forward)NNNN)r6   r7   r8   r   rY   r   CLIP_VISION_INPUTS_DOCSTRINGr   r   r   r   r:   r   r   r   rn   rp   r#   r#   rc   r$   r   R  s   

    
r   zAThe vision model from CLIP without any head or projection on top.c                
       s   e Zd ZeZdZed fddZejdddZ	e
eeeeddeej ee ee ee eeef d
ddZ  ZS )CLIPVisionModelre   rO   c                    s"   t  | t|| _|   d S r   )rX   rY   r   vision_modelr   rb   rc   r#   r$   rY     s    
zCLIPVisionModel.__init__rD   c                 C   s
   | j jjS r   r   rm   r^   rI   r#   r#   r$   r     s    z$CLIPVisionModel.get_input_embeddingsr   Nr   c                 C   s&   |dk	r|n| j j}| j||||dS )a  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, CLIPVisionModel

        >>> model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32")
        >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```Nre   r   r   r   )rP   r   r   )rJ   re   r   r   r   r#   r#   r$   rn     s    zCLIPVisionModel.forward)NNNN)r6   r7   r8   r   r   main_input_namerY   r   r   r   r   r   r   r   r   r   r:   r   r   r   rn   rp   r#   r#   rc   r$   r     s"   
    
r   c                       s  e Zd ZeZed fddZeedee	j
 ee	j
 ee	j
 ee ee ee e	jdddZeedee	j ee ee ee e	jdd	d
Zeeeeeddee	j ee	j ee	j
 ee	j ee ee ee ee eeef d	ddZ  ZS )r   rO   c                    s   t  | t|jts.tdt|j dt|jtsPtdt|j d|j}|j}|j	| _	|j
| _|j
| _t|| _t|| _tj| j| j	dd| _tj| j| j	dd| _tt| jj| _|   d S )NzKconfig.text_config is expected to be of type CLIPTextConfig but is of type .zOconfig.vision_config is expected to be of type CLIPVisionConfig but is of type FrQ   )rX   rY   r   text_configr   r|   typevision_configr   projection_dimrZ   r   r   r   r   r   r   r   r   r   r   r\   r   r   rP   Zlogit_scale_init_valuelogit_scaler   )rJ   rP   r   r   rc   r#   r$   rY     s(    

zCLIPModel.__init__Nr   c           
      C   sh   |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}| j||||||d}|d }| |}	|	S )a  
        Returns:
            text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
            applying the projection layer to the pooled output of [`CLIPTextModel`].

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, CLIPModel

        >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> text_features = model.get_text_features(**inputs)
        ```Nr   r   )rP   r   r   r   r   r   )
rJ   rt   r   rS   r   r   r   text_outputsr   Ztext_featuresr#   r#   r$   get_text_features  s    	
zCLIPModel.get_text_featuresr   c                 C   sd   |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}| j||||d}|d }| |}|S )aD  
        Returns:
            image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
            applying the projection layer to the pooled output of [`CLIPVisionModel`].

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, CLIPModel

        >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
        >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> image_features = model.get_image_features(**inputs)
        ```Nr   r   )rP   r   r   r   r   r   )rJ   re   r   r   r   vision_outputsr   Zimage_featuresr#   r#   r$   get_image_features  s    
zCLIPModel.get_image_featuresr   )	rt   re   r   rS   return_lossr   r   r   r'   c	              	   C   s*  |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}| j||||d}	| j||||||d}
|	d }| |}|
d }| |}||jdddd }||jdddd }| j	
 }t|| | }| }d}|rt|}|s|||||
|	f}|dk	r|f| S |S t||||||
|	d	S )
a  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, CLIPModel

        >>> model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
        >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(
        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
        ... )

        >>> outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```Nr   r   r   rR   rU   T)r   rg   Zkeepdim)r?   r@   rA   r=   r2   rB   rC   )rP   r   r   r   r   r   r   r   Znormr   expr   matmulr/   r0   r>   )rJ   rt   re   r   rS   r   r   r   r   r   r   r2   r=   r   rA   r@   r?   outputr#   r#   r$   rn   J  sT    &	


zCLIPModel.forward)NNNNNN)NNNN)NNNNNNNN)r6   r7   r8   r   r   rY   r   r   r   r   ro   r   r:   r   r   r   CLIP_INPUTS_DOCSTRINGr   r>   rw   r   r   rn   rp   r#   r#   rc   r$   r     sf          .    0
        
r   zf
    CLIP Text Model with a projection layer on top (a linear layer on top of the pooled output).
    c                       s   e Zd ZeZddgZed fddZejdddZ	d	d
 Z
eeeeeddeej eej eej ee ee ee eeef dddZ  ZS )r   rq   r   rO   c                    s8   t  | t|| _tj|j|jdd| _| 	  d S NFr   )
rX   rY   r   r   r   r   rZ   r   r   r   rb   rc   r#   r$   rY     s    
z$CLIPTextModelWithProjection.__init__rD   c                 C   s
   | j jjS r   r   rI   r#   r#   r$   r     s    z0CLIPTextModelWithProjection.get_input_embeddingsc                 C   s   || j j_d S r   r   r   r#   r#   r$   r     s    z0CLIPTextModelWithProjection.set_input_embeddingsr   Nr   c                 C   s   |dk	r|n| j j}| j||||||d}|d }| |}	|sj|	|d f|dd  }
tdd |
D S t|	|j|j|jdS )	a  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, CLIPTextModelWithProjection

        >>> model = CLIPTextModelWithProjection.from_pretrained("openai/clip-vit-base-patch32")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> text_embeds = outputs.text_embeds
        ```Nr   r   r   rR   c                 s   s   | ]}|d k	r|V  qd S r   r#   rG   r   r#   r#   r$   rK     s      z6CLIPTextModelWithProjection.forward.<locals>.<genexpr>)r=   r3   r4   r5   )	rP   r   r   r   rL   r<   r3   r4   r5   )rJ   rt   r   rS   r   r   r   r   r   r=   r   r#   r#   r$   rn     s(    	
z#CLIPTextModelWithProjection.forward)NNNNNN)r6   r7   r8   r   r   r   rY   r   r   r   r   r   r   r   r<   r   r   ro   r   r   r   rn   rp   r#   r#   rc   r$   r     s,   

      
r   zh
    CLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output).
    c                
       s   e Zd ZeZdZed fddZejdddZ	e
eeeeddeej ee ee ee eeef d
ddZ  ZS )r   re   rO   c                    s8   t  | t|| _tj|j|jdd| _| 	  d S r   )
rX   rY   r   r   r   r   rZ   r   r   r   rb   rc   r#   r$   rY     s    
z&CLIPVisionModelWithProjection.__init__rD   c                 C   s
   | j jjS r   r   rI   r#   r#   r$   r     s    z2CLIPVisionModelWithProjection.get_input_embeddingsr   Nr   c           	      C   s|   |dk	r|n| j j}| j||||d}|d }| |}|sf||d f|dd  }tdd |D S t||j|j|jdS )	a  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, CLIPVisionModelWithProjection

        >>> model = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-base-patch32")
        >>> processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> image_embeds = outputs.image_embeds
        ```Nr   r   r   rR   c                 s   s   | ]}|d k	r|V  qd S r   r#   r   r#   r#   r$   rK   =  s      z8CLIPVisionModelWithProjection.forward.<locals>.<genexpr>)r2   r3   r4   r5   )	rP   r   r   r   rL   r1   r3   r4   r5   )	rJ   re   r   r   r   r   r   r2   r   r#   r#   r$   rn     s$    
z%CLIPVisionModelWithProjection.forward)NNNN)r6   r7   r8   r   r   r   rY   r   r   r   r   r   r   r1   r   r   r:   r   r   r   rn   rp   r#   r#   rc   r$   r     s"   

    
r   )N)r   )@r9   dataclassesr   typingr   r   r   r   r   Ztorch.utils.checkpointr   Zactivationsr	   Zmodeling_outputsr
   r   Zmodeling_utilsr   r   r   r   r   r   r   Zconfiguration_clipr   r   r   Z
get_loggerr6   loggerZ_CHECKPOINT_FOR_DOCZ"CLIP_PRETRAINED_MODEL_ARCHIVE_LISTro   r   r   r%   r-   r0   r1   r<   r>   r   rN   rq   rx   r   r   r   ZCLIP_START_DOCSTRINGr   r   r   r   Sizer)   r   r   r   r   r   r   r   r   r#   r#   r#   r$   <module>   s   
$#!h2D &i    ]9:6 bK