U ,-e@sdZddlZddlmZmZmZmZddlZddlm m Z ddl Zddlm Z ddl mZmZmZddlmZddlmZmZmZmZmZmZmZdd lmZdd lmZdd lm Z m!Z!m"Z"m#Z#m$Z$d d l%m&Z&e#'e(Z)dZ*dZ+dgZ,Gddde j-Z.Gddde j-Z/Gddde j-Z0Gddde j-Z1Gddde j-Z2Gddde j-Z3Gddde j-Z4e5e4Gddde j-Z6Gd d!d!e j-Z7Gd"d#d#e j-Z8Gd$d%d%e j-Z9Gd&d'd'e j-Z:Gd(d)d)e j-Z;Gd*d+d+eZe!d.e=Gd/d0d0e<Z?e!d1e=Gd2d3d3e<Z@e!d4e=Gd5d6d6e<ZAe!d7e=Gd8d9d9e<ZBe!d:e=Gd;d<dd?d?e<ZDGd@dAdAe j-ZEe!dBe=GdCdDdDe<ZFdS)EzPyTorch MEGA model.N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN),BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)ALL_LAYERNORM_LAYERS)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) MegaConfigzmnaylor/mega-base-wikitextrcs0eZdZdZedfdd ZdddZZS) MegaEmbeddingsz Mega's basic implementation does not incorporate token type embeddings, so this is a stripped-down version of RoBERTa's embeddings which optionally includes token types configcsrttj|j|j|jd|_|j|_ |j rft|j |j|_ |j dt j|jt jdddd|j|_dS)N) padding_idxtoken_type_idsdtype)rF) persistent)super__init__r Embedding vocab_size hidden_sizeZ pad_token_idword_embeddingsZadd_token_type_embeddingsuse_token_typesZtype_vocab_sizetoken_type_embeddingsregister_buffertorchzeros max_positionslongexpandrselfr __class__g/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/mega/modeling_mega.pyr&As zMegaEmbeddings.__init__Nc Cs|dkr|dkrtdn8|dk r<|}|j}||}n|dd}|j}|jr|dkrt|dr|jddd|df}||d|d}|}ntj |tj |d}| |}||} n|} | S)Nz.Must provide one of input_ids or inputs_embedsr#r rrr"device) ValueErrorsizer:r*r+hasattrr r2r.r/r1r,) r4 input_idsr inputs_embeds input_shaper:Zbuffered_token_type_idsZ buffered_token_type_ids_expandedr,Z embeddingsr7r7r8forwardOs&     zMegaEmbeddings.forward)NNN__name__ __module__ __qualname____doc__rr&rA __classcell__r7r7r5r8r;srcs.eZdZdZedfdd ZddZZS) MegaSimpleRelativePositionalBiaszs Simple relative positional embeddings copied from the Mega repo; renamed variables for better readability rcsNt||_|jjdkr$|jjn|jj|_ttd|jd|_ dS)Nrr) r%r&r chunk_sizer0r Parameterr.Tensor rel_pos_biasr3r5r7r8r&vs z)MegaSimpleRelativePositionalBias.__init__cCs||jkrtd||j|j|j||j|d}t|d|f}t||f}|d| }||d|d}d|dd}| d|}|dd||f}|S)Nz-Sequence length {} going beyond max length {}rrr rI) r0r;formatrMFpadr.tileviewr<)r4seq_lenbiasrQstartendr7r7r8rA|s z(MegaSimpleRelativePositionalBias.forwardrBr7r7r5r8rHqsrHcsJeZdZdZedfdd ZeeedddZdd Z d d Z Z S) MegaRotaryRelativePositionalBiasap Rotary relative bias for positional information; similar in concept to RoPE (i.e. RoFormer) but taken from the Mega repo due to differences in implementation. When initialized, produces a positional bias which ranges from position 0 to config.max_positions, but can extrapolate to longer sequences. Can be indexed according to input position IDs rcst|jddkr td||_|j|_|jjdkrB|jjn|jj|_t |j|j\|_ |_ t td|j|_t td|j|_|dtdgdS)NrIrzCRotary positional bias requires `hidden_size` to be a multiple of 2r _float_tensor)r%r&r) RuntimeErrorrshared_representation_size embed_dimrJr0rWget_sinusoid_embeddingssinecosinerrKr.rLalphab_paramr- FloatTensorr3r5r7r8r&s  z)MegaRotaryRelativePositionalBias.__init__)r0 embedding_dimcCsf|d}td|}ttj|tjd| }tj|tjdd|d}t|t|fS)NrIi'r!rr) mathlogr.exparangefloat unsqueezesincos)r0rcZhalf_dimZembr7r7r8r]s  z8MegaRotaryRelativePositionalBias.get_sinusoid_embeddingscCs|\}}tj|ddd\}}|jdks:||jdkrTt||\|_|_||_|j|j |_|j|j |_|jd|}|jd|}tj ||||||||gddS)NrIr#dimrr) r<r.chunkr^rWr]r_r0torXcat)r4inputrSr\Zchunk_1Zchunk_2rjrkr7r7r8rotarys z'MegaRotaryRelativePositionalBias.rotarycCs>||j||j}||j||j}td||}|S)Nz mk,nk->mn)rrr`r2r\rar.einsum)r4rSZ rotary_alphaZ rotary_betarTr7r7r8rAsz(MegaRotaryRelativePositionalBias.forward) rCrDrErFrr& staticmethodintr]rrrArGr7r7r5r8rWs  rWcs2eZdZdZdfdd Zd edddZZS) MegaDropoutab A unified class for standard dropout functionality and featurewise dropout. The original fairseq Mega repo used 2 classes for these, which included some unnecessary handling of training logic and an unused `inplace` option. The original implementation used torch.nn.functional instead of submodules, which is retained here as well. Fcst||_||_dSN)r%r&dropout_probabilityis_featurewise)r4rxryr5r7r8r&s zMegaDropout.__init__ batch_firstcCs|jrl|r.tj|dd|j|jdddS|dkrBtdtj|ddd|j|jddddSntj ||j|jdSdS) Nr#)ptrainingr zzFeature dropout inputs must be exactly 3-dimensional if inputs are ordered [sequence length, batch size, hidden dimension]rrIr) ryrOZ dropout2d transposerxr~rmr;permutedropout)r4rqr{r7r7r8rAs(  zMegaDropout.forward)F)F)rCrDrErFr&boolrArGr7r7r5r8rvsrvcs*eZdZdZdfdd ZddZZS) MegaRMSNormz RMSNorm used in Mega implementation. Differs from T5's RMSNorm by applying the weight prior to taking the square root (as opposed to after in T5) ư>TcsFt||_||_||_|r6tt|j|_ n | dddS)Nweight) r%r&Z num_featuresepsaffinerrKr.rLrregister_parameter)r4Znumber_featuresrrr5r7r8r&s zMegaRMSNorm.__init__cCsBtjt|ddd}|jdk r*||j}|t||j|S)Nr#Trmkeepdim)r.meansquarerrsqrtr)r4rq mean_squarer7r7r8rAs   zMegaRMSNorm.forward)rTrCrDrErFr&rArGr7r7r5r8rs rcs*eZdZdZdfdd ZddZZS) MegaScaleNormz Scale normalization introduced in MEGA which is similar to RMSNorm, but uses a single parameter for scalar multiplication instead of a vector, and applies over a specified dimension rTcsDt||_||_||_|r4ttd|_ n | dddS)Nrscalar) r%r&rmrrrrKr.rLrr)r4rmrrr5r7r8r& s zMegaScaleNorm.__init__cCsDtjt||jdd}|jdk r,|j|}|t||j}|S)NTr)r.rrrmrrr)r4rqroutputr7r7r8rAs   zMegaScaleNorm.forward)rTrr7r7r5r8rs rcs*eZdZdZd fdd ZddZZS) MegaSequenceNormz A wrapper class for various layer normalization options used in Mega. Used to handle differences in expectations on input axis locations for different normalization methods. h㈵>TFcst|dkr&tj|||d|_nz|dkr@td||d|_n`|dkrZt|||d|_nF|dkrvtj|||d|_n*|d krtj|||d|_nt d |dS) NZ layernorm)Zelementwise_affineZ scalenormr#)rmrrZrmsnorm)rr batchnormZ syncbatchnormzUnknown norm type: {}) r%r&r LayerNormnormrrZ BatchNorm1dZ SyncBatchNormr;rN)r4Z norm_typercrrZexportr5r7r8r&"s zMegaSequenceNorm.__init__cCsZt|jtjjjrL|dkr&td|ddd}||}|dddS||SdS)Nr z.BatchNorm inputs must be exactly 3-dimensionalrrIr) isinstancerrmodulesrZ _BatchNormrmr;r)r4rqr7r7r8rA1s  zMegaSequenceNorm.forward)rTFrr7r7r5r8rsrcseZdZdZedfdd ZddZeddd Zd d Z edd d Z ddZ dddZ dddZ deejeejeejdddZZS)MegaMultiDimensionDampedEmaa Mega's Exponential Moving Average layer, largely left unmodified from the original repo with the exception of variable names and moving away from the stateful representation of incremental decoding state. See "https://arxiv.org/abs/2209.10655" for more details. rcst||_|j|_|j|_|j|_|j|_t d|j|_ |jrRd|jn|j}t t||jd|_t t||jd|_t t||jd|_t t||j|_t t|j|_d|_d|_dS)N?rIr)r%r&rr)r\ema_projection_sizendim bidirectional truncationrdsqrtscalerrKr.rLdamping_factor decay_factorema_expansion_matrixkernel_projection_matrixresidual_weight_kernel_coeffs)r4rZ kernel_dimr5r7r8r&Gs z$MegaMultiDimensionDampedEma.__init__cCs2d|_t|j}t|j}d||}||fS)Nr)rr.sigmoidrr)r4rrprevious_timestep_weightr7r7r8_compute_ema_coefficients_s    z5MegaMultiDimensionDampedEma._compute_ema_coefficientslengthcCs`d|_|\}}t||dd|t|}||jt|}t d||j |j S)Nr dnl,dn->dl) rrr.rgrorRrerrfrsrr)r4rrrvanderkernelr7r7r8_compute_efficient_ema_kernelgs  $z9MegaMultiDimensionDampedEma._compute_efficient_ema_kernelcCs,|jr|S|jdkr"||_|jSdSrw)r~rrr4r7r7r8get_ema_coefficientsss   z0MegaMultiDimensionDampedEma.get_ema_coefficientscCsf|jdkr|n t|j|}|jr*||S|jdksD|jd|krP|||_|jdd|fSdS)Nr#.)rminr~rrr<)r4r kernel_sizer7r7r8get_ema_kernel{s   z*MegaMultiDimensionDampedEma.get_ema_kernelcCsLtjj|d|d}tjj|d|d}tjj||d|d}|S)NrI)n)r.ZfftZrfftrhZirfft)r4inputsrrZ inputs_fftZ kernel_fftZconvolved_sequencer7r7r8fft_convolutionsz+MegaMultiDimensionDampedEma.fft_convolutionNcCsr|dkr|j||dS|\}}t|d|dd|dt|}t|}|dk r|ddddddf|j|j  d}t d||}|dddddf|} nd}d} |ddddddf}||j |} t d| |j|j } |j || |ddd|f} | |} |dk r4| |} t d |tj| d gd } | dk r`| | } | d dd| fS) Nr past_stater#z bdn,dnl->bdlrr.rz bdl,dnl->bdnrI)Zdims) one_ema_steprr.rgrorRrerfrrrirsrrtype_asflipr)r4rrrrrrZ past_ema_projZpast_ema_stateZpast_vandermonderZ kernel_proj ema_outputZupdated_hidden_stater7r7r8ema_steps0 $ ,   z$MegaMultiDimensionDampedEma.ema_stepcCs^|\}}||jd|}|dk r:||d|}td||j|j}|d|fS)Nr#z bdn,dn->bdr)rrsqueezer.rsrrri)r4rrrr updated_stateoutr7r7r8rs  z(MegaMultiDimensionDampedEma.one_ema_stepF)attention_mask prev_state use_cachereturncCs|\}}}||jkr.td|d|j||j}|ddd}|dk rb||d|}|jrt|rttd|r|j |||d\} } t | |} | | fS| |} |} d} | d}|jr