U ,-er@sbdZddlZddlmZmZmZmZddlZddlm m Z ddl Zddlm Z ddl mZmZmZddlmZddlmZmZmZdd lmZdd lmZdd lmZmZmZmZm Z d d l!m"Z"erddl#m$Z$m%Z%ddl&m'Z'm(Z(m)Z)e*e+Z,dZ-ddZ.dej4e2ej4d'd(d)Z?Gd*d+d+e j6Z@Gd,d-d-e@ZAGd.d/d/e j6ZBd0ZCed1eCGd2d3d3eZDd4ZEed1eCGd5d6d6eDZFGd7d8d8eDZGed9eCGd:d;d;eDZHdS)>z PyTorch LLaMA model.N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)PreTrainedModel)ALL_LAYERNORM_LAYERS)add_start_docstrings%add_start_docstrings_to_model_forwardis_flash_attn_availableloggingreplace_return_docstrings) LlamaConfig)flash_attn_funcflash_attn_varlen_func)index_first_axis pad_input unpad_inputrcCsX|jdtjd}tj|dd}|}ttj |dtjjdd}|||fS)NdimdtypeF)as_tupler)rr) sumtorchint32ZnonzeroflattenmaxitemFpadcumsum) padding_maskZseqlens_in_batchindicesZmax_seqlen_in_batchZ cu_seqlensr-i/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/llama/modeling_llama.py_get_unpad_data6s r/)input_ids_shaper devicepast_key_values_lengthcCs|\}}tj||ft|j|d}tj|d|d}|||d|ddkd||}|dkrtj tj ||||d|gdd}|ddddddf |d|||S)zB Make causal mask used for bi-directional self-attention. r1rrrr r1rN) r#fullfinfominarangesize masked_fill_viewtocatZzerosexpand)r0r r1r2bsztgt_lenmaskZ mask_condr-r-r._make_causal_maskCs"  rC)rBr rAcCsj|\}}|dk r|n|}|ddddddf|d|||}d|}||tjt|jS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. Nr?)r:r?r=Z masked_fillr#boolr7r8)rBr rAr@Zsrc_lenZ expanded_maskZ inverted_maskr-r-r. _expand_maskUs  *rFcs&eZdZdfdd ZddZZS) LlamaRMSNormư>cs&ttt||_||_dS)z; LlamaRMSNorm is equivalent to T5LayerNorm N)super__init__r Parameterr#onesweightvariance_epsilon)self hidden_sizeeps __class__r-r.rJds zLlamaRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)NrT)Zkeepdim) r r=r#float32powmeanZrsqrtrNrM)rO hidden_states input_dtypeZvariancer-r-r.forwardls  zLlamaRMSNorm.forward)rH__name__ __module__ __qualname__rJrZ __classcell__r-r-rRr.rGcsrGcs0eZdZd fdd ZddZd dd ZZS) LlamaRotaryEmbedding'Ncsrt||_||_||_d|jtd|jd||j}|j d|dd|j ||j j t ddS)NrDrrTinv_freqF persistentseq_lenr1r )rIrJrmax_position_embeddingsbaser#r9floatr=register_buffer_set_cos_sin_cachercr1Zget_default_dtype)rOrrhrir1rcrRr-r.rJxs *zLlamaRotaryEmbedding.__init__cCs||_tj|j||jjd}td||j}tj||fdd}|jd|ddddddf |dd|jd| ddddddf |dddS Nr1r i,j->ijrr5 cos_cachedFrd sin_cached) max_seq_len_cachedr#r9rcr einsumr>rkcosr=sinrOrgr1r tfreqsembr-r-r.rls .z'LlamaRotaryEmbedding._set_cos_sin_cachecCsn||jkr|j||j|jd|jddddd|dfj|jd|jddddd|dfj|jdfS)Nrf.)r )rrrlr1r rpr=rq)rOxrgr-r-r.rZs  &&zLlamaRotaryEmbedding.forward)rarbN)N)r\r]r^rJrlrZr_r-r-rRr.r`ws r`cs*eZdZdZd fdd Zdd ZZS) !LlamaLinearScalingRotaryEmbeddingz[LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendevrarbNrDcs||_t||||dSNscaling_factorrIrJrOrrhrir1r~rRr-r.rJsz*LlamaLinearScalingRotaryEmbedding.__init__cCs||_tj|j||jjd}||j}td||j}tj||fdd}|jd| ddddddf |dd|jd| ddddddf |dddSrm) rrr#r9rcr r~rsr>rkrtr=rurvr-r-r.rls .z4LlamaLinearScalingRotaryEmbedding._set_cos_sin_cache)rarbNrDr\r]r^__doc__rJrlr_r-r-rRr.r{sr{cs*eZdZdZd fdd Zdd ZZS) %LlamaDynamicNTKScalingRotaryEmbeddingzmLlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozillararbNrDcs||_t||||dSr|r}rrRr-r.rJsz.LlamaDynamicNTKScalingRotaryEmbedding.__init__c Cs||_||jkrx|j|j||j|jd|j|jd}d|td|jd||j}|j d|ddtj|j||j j d}t d ||j }tj ||fd d }|j d |ddddddf|dd|j d |ddddddf|dddS)NrrTrDrrcFrdrnrorr5rprq)rrrhrir~rr#r9rjr=rkrcr rsr>rtru) rOrgr1r rircrwrxryr-r-r.rls (.z8LlamaDynamicNTKScalingRotaryEmbedding._set_cos_sin_cache)rarbNrDrr-r-rRr.rsrcCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..NrrTr5)shaper#r>)rzx1Zx2r-r-r. rotate_halfsrcCsl|dd}|dd}||d}||d}||t||}||t||}||fS)Nrr)squeeze unsqueezer)qkrtru position_idsZq_embedZk_embedr-r-r.apply_rotary_pos_embsrcs$eZdZfddZddZZS)LlamaMLPcsrt||_|j|_|j|_tj|j|jdd|_tj|j|jdd|_tj|j|jdd|_ t |j |_ dSNFbias) rIrJconfigrPintermediate_sizerLinear gate_projup_proj down_projr Z hidden_actact_fnrOrrRr-r.rJs zLlamaMLP.__init__cs|jjdkr|j|jj}|jjj|dd|jjj|dd|jjj|ddtj fddt |jjDdd}tj fddt |jjDdd}| ||j|ddfd dt |jjD}t |}n || ||}|S) Nrrr5csg|]}t|qSr-r(linear.0i)gate_proj_slicesrzr-r. sz$LlamaMLP.forward..rcsg|]}t|qSr-rr)up_proj_slicesrzr-r.rsrTcs g|]}t||qSr-rr)down_proj_slicesintermediate_statesr-r.rs) rpretraining_tprrrMsplitrrr#r>rangerr")rOrzslicerrrr-)rrrrrzr.rZs" &    zLlamaMLP.forwardr[r-r-rRr.rs r)rXn_repreturncCs^|j\}}}}|dkr|S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)rr?reshape)rXrbatchnum_key_value_headsslenhead_dimr-r-r. repeat_kvs 0rcseZdZdZedfdd ZddZeje e ddd Z deje eje ej e e ejeee ej e eje eje e ejfd d dZZS)LlamaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrcst||_|j|_|j|_|j|j|_|j|_|j|j|_|j |_ |j |_ |j|j|jkrt d|jd|jdt j |j|j|j|jd|_t j |j|j|j|jd|_t j |j|j|j|jd|_t j |j|j|j|jd|_|dS)Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r)rIrJrrPZnum_attention_heads num_headsrrnum_key_value_groupsrh rope_theta ValueErrorrrZattention_biasq_projk_projv_projo_proj _init_roperrRr-r.rJ s$ zLlamaAttention.__init__cCs|jjdkr$t|j|j|jd|_nj|jjd}|jjd}|dkr^t|j|j||jd|_n0|dkrt|j|j||jd|_nt d|dS)N)rhritypefactorr)rhr~riZdynamiczUnknown RoPE scaling type ) rZ rope_scalingr`rrhr rotary_embr{rr)rOZ scaling_typer~r-r-r.r!s.      zLlamaAttention._init_rope)tensorrgr@cCs ||||j|jddS)NrrT)r<rr transpose contiguous)rOrrgr@r-r-r._shape<szLlamaAttention._shapeNFrXattention_maskrpast_key_valueoutput_attentions use_cacher+rcs\}} } |jjdkr|j|j|jj} |jjj|j|j|jjdd|j jj| dd|j jj| ddfddt |jjD} t j | dd} fddt |jjD} t j | dd} fddt |jjD}t j |dd}n|} | } | }| || |j|jdd } | || |j|jdd } ||| |j|jdd }| jd }|dk r||djd 7}|j||d \}}t| | |||\} } |dk rt j |d| gd d} t j |d|gd d}|r| |fnd}t| |j} t||j}t | | d d t|j}|||j| |fkrltd ||j| |fd||dk r||d| |fkrtd|d| |fd|||}tjj|dt jd| j}t ||||j| |jfkrtd||j| |jfddd  || |j!|jjdkrj|j!|jjd d|j"jj|j!|jjddt#fddt |jjDn |"|sd}||fS)Nrrr5csg|]}t|qSr-rr)rX query_slicesr-r.rSsz*LlamaAttention.forward..rcsg|]}t|qSr-rr)rX key_slicesr-r.rVscsg|]}t|qSr-rr)rX value_slicesr-r.rYsrTrgr z$Attention weights should be of size z , but is z!Attention mask should be of size rz `attn_output` should be of size cs g|]}t||qSr-rr) attn_output o_proj_slicesr-r.rs)$r:rrrrrrMrrrrrr#r>r<rrrrrrmatmulmathsqrtrr functionalZsoftmaxrUr=r rrrPrr")rOrXrrrrrr+r@q_len_Zkey_value_slicing query_states key_states value_states kv_seq_lenrtru attn_weightsr-)rrXrrrrr.rZ?sv             " zLlamaAttention.forward)NNNFFN)r\r]r^rrrJrr#Tensorintrr LongTensorrrErZr_r-r-rRr.rs( rc@szeZdZdZd ejeejeejeeeje e eejeejeejeeejfdddZ d dd Z d d Z dS)LlamaFlashAttention2aF Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. NFrcCsd}|\}} } ||} ||} ||} | || |j|jdd} | || |j|jdd} | || |j|jdd} | j d}|dk r||dj d7}|j | |d\}}t | | |||\} } |dk rt j |d| gdd} t j |d| gdd} |r| | fnd}| dd} | dd} | dd} d}| j}|t jkr~td | t j} | t j} | t j} |j| | | || |d }||| |j}||}|sd}|||fS) NFrrTrrrr5zThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in float16.)dropout)r:rrrr<rrrrrrrr#r>r rUlogger warning_oncer=Zfloat16_flash_attention_forwardrrPrr)rOrXrrrrrr+r@rrrrrrrtruZ dropout_raterYrrr-r-r.rZsT              zLlamaFlashAttention2.forwardrc Cs|dk rl|jd}||||||\}}}} } } | \} } | \}}t|||| | ||||dd }t|| ||}nt|||||dd}|S)a Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token first unpad the input, then computes the attention scores and pad the final attention scores. Args: query_states (`torch.Tensor`): Input query states to be passed to Flash Attention API key_states (`torch.Tensor`): Input key states to be passed to Flash Attention API value_states (`torch.Tensor`): Input value states to be passed to Flash Attention API padding_mask (`torch.Tensor`): The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the position of padding tokens and 1 for the position of non-padding tokens. dropout (`int`, *optional*): Attention dropout softmax_scale (`float`, *optional*): The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim) NrT) cu_seqlens_q cu_seqlens_kZ max_seqlen_qZ max_seqlen_kZ dropout_p softmax_scalecausal)rr)r _upad_inputrrr)rOrrrr+ query_lengthrr batch_size indices_qZ cu_seq_lensZ max_seq_lensrrmax_seqlen_in_batch_qmax_seqlen_in_batch_kZattn_output_unpadrr-r-r.rsB  z-LlamaFlashAttention2._flash_attention_forwardcCst|\}}}|j\} } } } t|| | | | |}t|| | | | |}|| kr|t|| | |j| |}|} |}|}nd|dkrd}tj| dtj|jd} | dd}| d}n(|dd| df}t ||\}}} }||||| |f||ffS)Nrr4r) r/rrrrr#r9r$r1rr)rOZ query_layerZ key_layerZ value_layerr+rZ indices_krrrrrrrrrr-r-r.r*sH  z LlamaFlashAttention2._upad_input)NNNFFN)rN) r\r]r^rr#rrrrrErZrrr-r-r-r.rs*   P 5rcseZdZedfdd Zd ejeejeejee ejee ee eeje ej ee ej ej ffdddZ Z S) LlamaDecoderLayerrcsft|j|_t|dds(t|dnt|d|_t||_t |j|j d|_ t |j|j d|_ dS)NZ_flash_attn_2_enabledFrrQ) rIrJrPgetattrrr self_attnrmlprG rms_norm_epsinput_layernormpost_attention_layernormrrRr-r.rJRs   zLlamaDecoderLayer.__init__NFrc Csz|}||}|j|||||||d\}} } ||}|}||}||}||}|f} |rh| | f7} |rv| | f7} | S)a] Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states )rXrrrrrr+)rrrr) rOrXrrrrrr+ZresidualZself_attn_weightsZpresent_key_valueoutputsr-r-r.rZ^s,      zLlamaDecoderLayer.forward)NNNFFN)r\r]r^rrJr#rrrrrE FloatTensorrZr_r-r-rRr.rQs" raJ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`LlamaConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. zSThe bare LLaMA Model outputting raw hidden-states without any specific head on top.c@s8eZdZeZdZdZdgZdZdZ ddZ d dd Z d S) LlamaPreTrainedModelmodelTrpast_key_valuescCs||jj}t|tjr>|jjjd|d|jdk rx|jj n:t|tj rx|jjjd|d|j dk rx|jj|j  dS)Nr)rWstd) rZinitializer_range isinstancerrrMdataZnormal_rZzero_ Embedding padding_idx)rOmodulerr-r-r. _init_weightss    z"LlamaPreTrainedModel._init_weightsFcCst|tr||_dSr|)r LlamaModelgradient_checkpointing)rOrvaluer-r-r._set_gradient_checkpointings z0LlamaPreTrainedModel._set_gradient_checkpointingN)F) r\r]r^r config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementZ_supports_flash_attn_2rrr-r-r-r.rs ra> Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cseZdZdZedfdd ZddZddZd d Ze e de j e e je e j e ee je e je ee ee ee eeeefd d dZZS)rz Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] Args: config: LlamaConfig rcsvtj|_j|_tjj|j|_t fddt j D|_ t jjd|_d|_|dS)Ncsg|] }tqSr-)r)rrrr-r.rsz'LlamaModel.__init__..rF)rIrJ pad_token_idr vocab_sizerrrP embed_tokensZ ModuleListrZnum_hidden_layerslayersrGrnormr post_initrrRrr.rJs  zLlamaModel.__init__cCs|jSr|rrOr-r-r.get_input_embeddingsszLlamaModel.get_input_embeddingscCs ||_dSr|r rOrr-r-r.set_input_embeddings szLlamaModel.set_input_embeddingscCs`d}|ddkr$t||j|j|d}|dk r\t||j|dd|j}|dkrT|n||}|S)Nrr)r1r2)rA)rCr r1rFr=)rOrZ input_shape inputs_embedsr2Zcombined_attention_maskZexpanded_attn_maskr-r-r._prepare_decoder_attention_mask$s z*LlamaModel._prepare_decoder_attention_maskN) input_idsrrrrrroutput_hidden_states return_dictrc  sdk r n|jj|dk r |n|jj}|dk r4|n|jj}| dk rH| n|jj} |dk rj|dk rjtdn2|dk r~|j\} } n|dk r|j\} } } ntd| } d}|dk r|ddjd}| |} |dkr|dk r|jn|j}tj || |tj |d}| d d| }n| d|  }|dkr4| |}|dkr\tj| | ftj|jd}dnd|krl|nd||| | f||}|}|jr|jr|rtdd}|rd nd}rd nd}|rd nd}t|jD]\}}|r||f7}|dk r ||nd|jrH|jrHfd d }tjj|||||}n|||||d }|d}|r||rzdnd f7}r||d f7}q||}|r||f7}|r|nd}| stdd||||fDSt||||dS)NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrrTr4rzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr-csfdd}|S)Ncs|fdiS)Nr+r-)inputs)rrr+rr-r.custom_forwardszILlamaModel.forward..create_custom_forward..custom_forwardr-)rrrr+r)rr.create_custom_forwardsz1LlamaModel.forward..create_custom_forward)rrrrrr+rcss|]}|dk r|VqdSr|r-)rvr-r-r. sz%LlamaModel.forward..)Zlast_hidden_staterrX attentions)rrrruse_return_dictrrr1r#r9longrr<rrLrErrZtrainingrr enumeraterutils checkpointrtupler )rOrrrrrrrrrrZ seq_lengthrZseq_length_with_pastr2r1rXZall_hidden_statesZall_self_attnsZnext_decoder_cacheidxZ decoder_layerrZ layer_outputsZ next_cacher-rr.rZ;s            zLlamaModel.forward) NNNNNNNNN)r\r]r^rrrJr r rrLLAMA_INPUTS_DOCSTRINGr#rrrrrrErrr rZr_r-r-rRr.rs6   rcseZdZdgZfddZddZddZdd Zd d Zd d Z ddZ e e e eeddejeejeejeeejeejeejeeeeeeeeeeefd ddZdddZeddZZS)LlamaForCausalLMzlm_head.weightcs@t|t||_|j|_tj|j|jdd|_| dSr) rIrJrrrrrrPlm_headrrrRr-r.rJs   zLlamaForCausalLM.__init__cCs|jjSr|rrr r-r-r.r sz%LlamaForCausalLM.get_input_embeddingscCs ||j_dSr|r$r r-r-r.r sz%LlamaForCausalLM.set_input_embeddingscCs|jSr|r#r r-r-r.get_output_embeddingssz&LlamaForCausalLM.get_output_embeddingscCs ||_dSr|r%)rOZnew_embeddingsr-r-r.set_output_embeddingssz&LlamaForCausalLM.set_output_embeddingscCs ||_dSr|r)rOdecoderr-r-r. set_decoderszLlamaForCausalLM.set_decodercCs|jSr|r(r r-r-r. get_decoderszLlamaForCausalLM.get_decoder) output_typerN rrrrrlabelsrrrrrc  s|dk r |n|jj}| dk r | n|jj} | dk r4| n|jj} |j|||||||| | d } | d|jjdkr|jjj|j |jjddfddt |jjD} t j | dd} n |} | } d} |dk r8| d ddddf}|d ddf}t}|d|jj }|d}||j}|||} | sh| f| dd}| dk rd| f|S|St| | | j| j| jd S) a Args: labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Returns: Example: ```python >>> from transformers import AutoTokenizer, LlamaForCausalLM >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rrrrrrrrrrrr5csg|]}t|qSr-rrrXZlm_head_slicesr-r.rsz,LlamaForCausalLM.forward..r.losslogitsrrXr)rrrrrrr#rMrrrr#r>rjrrr<r=r1r rrXr)rOrrrrrr.rrrrrr2r1Z shift_logitsZ shift_labelsloss_fctoutputr-r/r.rZsR)       zLlamaForCausalLM.forwardcKs|r|ddddf}|dd}|dk rp|dkrp|dd}||dkd|rp|dddfd}|dk r|dkrd|i}nd|i}||||d|d|S) Nrrrrrrr)rrrr)getrr*r;rupdate)rOrrrrkwargsrZ model_inputsr-r-r.prepare_inputs_for_generation<s&  z.LlamaForCausalLM.prepare_inputs_for_generationcs.d}|D] }|tfdd|Df7}q|S)Nr-c3s"|]}|d|jVqdS)rN)Z index_selectr=r1)rZ past_statebeam_idxr-r.r_sz2LlamaForCausalLM._reorder_cache..)r)rr:Zreordered_pastZ layer_pastr-r9r._reorder_cacheZs zLlamaForCausalLM._reorder_cache) NNNNNNNNNN)NNN)r\r]r^Z_tied_weights_keysrJr r r&r'r*r+rr!rr _CONFIG_FOR_DOCr#rrrrrrErrrZr8 staticmethodr;r_r-r-rRr.r"sN    ] r"a The LLaMa Model transformer with a sequence classification head on top (linear layer). [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). cseZdZfddZddZddZeed ej e ej e ej e e ej e ej e ej e ee ee ee eeeefd d d ZZS) LlamaForSequenceClassificationcs@t||j|_t||_tj|j|jdd|_| dSr) rIrJ num_labelsrrrrrPscorerrrRr-r.rJts   z'LlamaForSequenceClassification.__init__cCs|jjSr|r$r r-r-r.r }sz3LlamaForSequenceClassification.get_input_embeddingscCs ||j_dSr|r$r r-r-r.r sz3LlamaForSequenceClassification.set_input_embeddingsNr-c  Cs(| dk r | n|jj} |j|||||||| | d } | d} || } |dk rV|jd}n |jd}|jjdkr||dkr|td|jjdkrd}n4|dk rt||jj dd | j }nd}| tj || j d|f}d}|dk r| | j }|jjdkrR|jdkrd|j_n:|jdkrJ|jtj ks@|jtjkrJd |j_nd |j_|jjdkrt}|jdkr|||}n |||}nN|jjd krt}||d|j|d}n|jjd krt}|||}| s|f| dd}|dk r |f|S|St||| j| j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). N)rrrrrrrrrrz=Cannot handle batch sizes > 1 if no padding token is defined.rr3Z regressionZsingle_label_classificationZmulti_label_classificationr0)rrrr@rrrr#eqrZargmaxr=r1r9Z problem_typer?r rr rrr<rrrrXr)rOrrrrrr.rrrrZtransformer_outputsrXr2rZsequence_lengthsZ pooled_logitsr1r3r4r-r-r.rZsr         (    z&LlamaForSequenceClassification.forward) NNNNNNNNNN)r\r]r^rJr r rr!r#rrrrrrErrrrZr_r-r-rRr.r>ds6   r>)r)N)Irrtypingrrrrr#Ztorch.nn.functionalrrr(Ztorch.utils.checkpointZtorch.nnrrr Z activationsr Zmodeling_outputsr r rZmodeling_utilsrZ pytorch_utilsrrrrrrrZconfiguration_llamarZ flash_attnrrZflash_attn.bert_paddingrrrZ get_loggerr\rr<r/Sizer r1rrCrrFModulerGappendr`r{rrrrrrrrZLLAMA_START_DOCSTRINGrr!rr"r>r-r-r-r.sz        $ " 3F@9%