U
    d                     @   s4   d dl mZ d dlmZ d dlZG dd deZdS )    )schema)
ModelLayerNc                       s6   e Zd Zd fdd	Zd	d
 Zdd Zdd Z  ZS )LayerNormalizationlayer_normalizationN-C6?   T      ?c
                    s  t t| j|||f|
 t|tjs2td|| j	| _
|| _t| j
dksZtd| j
d }ttj| j
f| d| _| jd|gdd|	if|d	| _| jd
|gdddif|d	| _|| _| jr|| _n.t| j
dkstd|d| j t|| _d S )NzIncorrect input type: {}r   z&This layer supports only >= 2D tensorsr   outputscaleZConstantFillvalue)
param_nameshapeZinitializerZ	optimizerbiasg        z@When using alternative implementation, input data can only be 2Dz
%s_epsilon)superr   __init__
isinstancer   ZScalarAssertionErrorformatZ
field_typer   Zinput_shapeaxislennpZfloat32Zget_next_blob_referenceoutput_schemaZcreate_paramr
   r   use_layer_norm_opepsilonZmaybe_add_global_constantnamefloat)selfmodelinput_recordr   Zscale_optimZ
bias_optimr   r   r   Zscale_init_valuekwargsZ
input_dims	__class__ L/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/layers/layer_normalization.pyr      sP    
  



 zLayerNormalization.__init__c           	      C   s   | j  }| j }|d|d|dg}|j||| j| jd\}}}|j|| jg|dgd| jd}|j	|| j
g|d| jd d S )N	ln_outputln_meanln_stdev)r   r   	ln_scaledr   )	broadcastr   )r   field_blobsr   NextScopedBlobZ	LayerNormr   r   Mulr
   Addr   )	r   netZ
input_blobr$   Zoutput_blobs
normalizedZmeanZstdevZscaledr"   r"   r#   add_ops_with_layer_norm_opA   s,    


z-LayerNormalization.add_ops_with_layer_norm_opc                 C   sF  | d}| d}| d}| d}|| j |g |j|g|gdgd | d}|| j |g |g | d}||g|g | d	}||g|g | d
}	||| jg|	 | d}
|j	|	g|
gdd |j|
g|gdgd |
||g|g | d}||| jg|g ||| jg| j  d S )Nr$   r%   r&   ln_mean_arrr   )Zdimsln_centeredln_sqrln_sqr_meanln_var
ln_std_arrg      ?)exponentr'   )r*   ZReduceBackMeanr   r)   Z
ExpandDimsZSubZSqrr,   r   ZPowZDivr+   r
   r   r   )r   r-   r$   r%   r&   r0   r1   r2   r3   r4   r5   r'   r"   r"   r#   add_ops_without_layer_norm_op[   s*    









z0LayerNormalization.add_ops_without_layer_norm_opc                 C   s    | j r| | n
| | d S )N)r   r/   r7   )r   r-   r"   r"   r#   add_opsu   s    zLayerNormalization.add_ops)r   NNr   r   Tr   )__name__
__module____qualname__r   r/   r7   r8   __classcell__r"   r"   r    r#   r      s          4r   )Zcaffe2.pythonr   Zcaffe2.python.layers.layersr   Znumpyr   r   r"   r"   r"   r#   <module>   s   