U
    dV                     @   s   d dl mZmZ d dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZmZmZmZmZ d dlZd dlZd dlZd dlZd dlZd dlZeeZdd Zddd	Zd
d Zdd Zdd ZG dd deZ dS )    )FP16_ENGINES	Optimizer)get_current_scope)schema)	get_categorical_limitget_keyIdListIdScoreListIdListWithEvictedIdScoreListWithEvictedLayerPsParam
ModelLayeralmost_equal_schemasNc                 C   sf   t | tr^t| dr^td| j | jtkrFtd| j dS td| j dS ndS d S )Nenginez/Attempting to set trainer version for engine {}z"Setting FP16 trainer for engine {}fp16z"Setting FP32 trainer for engine {}fp32)
isinstancer   hasattrloggerinfoformatr   r   )Z	optim_def r   F/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/layers/sparse_lookup.py"get_trainer_version_based_on_optim    s    

r   c                 C   s   | dkst d| | dkr|d k	rz|d k	rz|d k	rz||k rVtd||| d} |d dkrtd|| d} ntd	|| |||| S )
N>   uint8rowwiser   fused_uint8rowwisefused_uint4rowwiser   -Unexpected version of sparse_lookup layer {0}r   zK{} fall back to uint8 because lookup table size {} < min_blob_size_4bits {}r         zM{} fall back to uint8 because lookup table dimension {} is not divisible by 2zWhen 4 bit quantization is enabled for {}, (i.e., Sparse lookup predictor version:{}), requires arguments blob_size:{}, min_blob_size_4bits:{}, embedding_dim:{})AssertionErrorr   r   r   
ValueError)version	blob_sizemin_blob_size_4bitsembedding_dimsparse_feature_namer   r   r   #get_sparse_lookup_predictor_version/   sL    
 r'   c                 C   s   | dkst d| | S )N>   r   r   r   )r    r   r"   r   r   r   !get_sparse_lookup_trainer_versionb   s    
r)   c                 C   s
   t | tS N)r   r   input_recordr   r   r   _is_id_listg   s    r-   c                 C   s   t | tddS )NFZcheck_field_types)r   r	   r+   r   r   r   _is_id_score_listk   s    r/   c                	       s   e Zd Zddddddddd	g	Zd
dddddd	gZdgZdddd
dgZd. fdd	Zdd Zdd Z	dd Z
d/ddZdd Zdd Zd0d!d"Zd#d$ Zd%d& Zd1d(d)Zd*d+ Zd,d- Z  ZS )2SparseLookupZ
LogMeanExpZ	LogSumExpZMaxMeanSumWeightedSumWeightedMeanSqrtNonePositionWeightedRecencyWeightedFloat16UniformFillNsparse_lookupF      ?c              	      s  t t| j|||f| t| j | _td| j  t|t	rF|g}t|t
spt|tsptdt|| j|dkrt| jstdt| j| j| j | _nt|dkrt| jstd| j| j | _nD|	rt| jstd| j|dkstd	|| j | _|| _|	| _t| j}|d
ksNtd| j||| _|g| | _t|| _|
| _|  }|p|| _d | _t| jt stj| jt!ddr| jj"| _| jdkrH| j| j#ks|	std| j| j#| jd
 dkrd| jd f| _| jd
 | j$ks6td| jd
 | j$|d ksHtd| jj%j&rb| jj%j&j'}nd }| j(d| j| j|t)| j|d|d| _*| jr| j(d|| j|j+d d| _,dddif| _-| j(dg | j-|j+d| _.t/t0j1|f| 2d| _3d S ) Nz"Setup the sparse lookup layer for zHUnexpected type for inner_shape, expected list or tuple, got {0} for {1}r7   z~PositionWeighted only support IdScoreList, but got {} for {}please use PositionWeighted layer to convert IdList to IdScoreListr8   zNRecencyWeighted only supports IdScoreList, while the sparse feature {} is not.zSUse_external_weights only supports IdScoreList, while the sparse feature {} is not.)r2   r3   zHUse_external_weights only supports Sum reducer, while the reducer is {}.r   z0{} should have categorical limit > 0, but got {}Fr.   r   a  Fp16 training is enabled. The reducer specified is not supported. Got {}. Supported reducers: {}. Right now, in general, sum, mean, positional pooling are supported. Attention is not. Please check if there is fp16 trained sparse features using advanced pooling.UniformFillr9   r   zjFp16 training is enabled. Init op for weight parameter must be fp16 compatibale. Got {}. Supported ops: {}z'Regularizer is not compatible with fp16w)
sparse_keyZaverage_length)
param_nameshapeinitializer	optimizerZps_paramregularizer
reinit_vec)r?   r@   rA   rB   rC   ZConstantFillvalueg        
scale_bias)r?   r@   rA   rB   output)4superr0   __init__r   r,   r>   r   r   r   intlisttupler    r   typer/   reprvaluesexternal_weightsreduceruse_external_weightsr   	input_dimr@   r   trainer_version#uniform_weight_init_scale_numerator_get_default_init_opweight_initevicted_valuesr   Zequal_schemasr
   r   Z_evicted_values_fp16_compatible_reducers_fp16_compatible_init_op_typeslengthsmetadataZexpected_valueZcreate_paramr   r=   ZNoOptimrD   Zscale_bias_initrF   ZScalarnpZfloat32Zget_next_blob_referenceoutput_schema)selfmodelr,   Zinner_shaperQ   rW   Zweight_optimnamerC   rR   rU   kwargsrS   Zdefault_init_opZ
avg_length	__class__r   r   rI      s    
  
    
 	
zSparseLookup.__init__c                 C   s   t tj| jd S )N   )	functoolsreduceoperatormulr@   r_   r   r   r   get_memory_usage  s    zSparseLookup.get_memory_usagec                 C   s   | j gS r*   )r=   rj   r   r   r   get_fp16_compatible_parameters
  s    z+SparseLookup.get_fp16_compatible_parametersc                 C   s$   t | jdks| jd dk r dS dS )Nr   r      FT)lenr@   rj   r   r   r   support_8bit  s    zSparseLookup.support_8bitTc                 C   sH   |   sg S |r(tdd}|| jgS tdd}|| j| jgS d S )NRowwiseQuantized8BitsWeightr=   zw, scale_bias)ro   collections
namedtupler=   rF   )r_   Zfusedrp   r   r   r   get_8bits_compatible_parameters  s      z,SparseLookup.get_8bits_compatible_parametersc                 C   s`   t | j| j }| jdkr.d| |df}n.| jdkrJd| |df}ntdt| j|S )Nr   r<   )minmaxr   r9   zATrain version {} is not currently supported for sparse feature {})mathsqrtrU   rS   rT   NotImplementedErrorr   r>   )r_   ZscaleZdefault_weight_initr   r   r   rV   "  s    

 z!SparseLookup._get_default_init_opc                 C   s   |dkr| | j|g|S |dkr@| | j|gd}|||S |dkr|| | j|gd}| | j|gd}|||g|S |dkr| | j|gd}|||S |dkr| | j|gd}|||S dd	|| j d S )
Nr   r   
gathered_wr   gathered_scale_biasr   r   z1Unsupported version of operators in SparseLookup !layer: {0} for sparse feature {1})	ZGatherr=   ZHalfToFloatrF   ZRowwise8BitQuantizedToFloatZ Fused8BitRowwiseQuantizedToFloatZ Fused4BitRowwiseQuantizedToFloatr   r>   )r_   netr"   
in_indicesoutry   rz   r   r   r   _gather_wrapper2  s6      zSparseLookup._gather_wrapperr   c           	      C   s   | j ||| j g}d| }|dkrj|dkrN|dkrN|j|| j |dd q|||| j |d n|dkr|t|| j	 ||d	 || j  nZ|d
kr||d || j  n6|dkr||d || j  ndd
|| j d S )NSparseLengthsr   r   r3   r   ZFP16)grad_on_weightsr   r   r   8BitsRowwiser   Fused8BitRowwiser   Fused4BitRowwise0Unsupported version of operator in SparseLookUp r{   )r=   r,   r[   ZSparseLengthsWeightedSumr^   field_blobs__getattr__insertrn   rF   r   r>   )	r_   r}   weightsrQ   r|   r"   r   op_input
layer_namer   r   r    _sparse_lengths_weighted_reducerQ  sR    
    z-SparseLookup._sparse_lengths_weighted_reducerc                 C   s  | j | jks td| j | j| j dkr$| j| j | j g}| j dkrVd| _ n| j dkrfd| _ d| j  }|dkr|	||| j
  n|d	kr|t|| j |	|d
 || j
  n\|dkr|	|d || j
  n8|dkr|	|d || j
  ndd|| j n| j dkrj|j| j g|dgdd}| | j |d|| n| j dkr| ||| j | j
  nX| ||| j d}|| j || j d }|	d| j  ||g| j
  d S )Nz&Unsupported reducer: {} for ID_LIST {})r2   r1   r3   r4   r3   r2   r4   r1   r   r   r   r   r   r   r   r   r   r{   r5   Zlengths_sqrtg      ?)powerr6   
table_rowsZ_sidZSortedSegmentRange)rQ   _id_list_supported_reducersr    r   r>   r=   r,   itemsr[   r   r^   r   r   rn   rF   ZLengthsToWeightsZNextScopedBlobr   r   ZLengthsToSegmentIds)r_   r|   r"   r   r   Zsqrt_weightr   Zsegment_idsr   r   r   _add_ops_id_list  s     


  
  

     zSparseLookup._add_ops_id_listc                 C   s  | j | jks td| j | j| j dkrN| | j | j | j || nB| j dks^| j	r| j| j | j
d||dd n| j dkrR| j| j | j g}d| j  }|d	kr|||| j  n|d
kr||d || j  n^|dkr||d || j  n8|dkr>||d || j  ndd|| j n>| j dkr|| ||| j | j  ndd| j | j d S )Nz,Unsupported reducer: {} for ID_SCORE_LIST {})r3   r4   )r7   r8   r3   r   r   )r2   r1   r   r   r   r   r   r   r   r   r   r{   r6   z9Only Sum, Mean, None are supported for IdScoreList input.z.Trying to create with {} for sparse feature {})rQ   !_id_score_list_supported_reducersr    r   r>   r   r,   keysrO   rR   rP   r=   r[   r   r^   r   r   )r_   r|   r"   r   r   r   r   r   _add_ops_id_score_list  s     
     

 
 
   z#SparseLookup._add_ops_id_score_listr   c                 C   sn   | j r*|r*|| j| j  | jg| jg t| jrD| j||d n&t| jr^| j	||d nd
| jd S )Nr(   zUnsupported input type {0})rX   ZCopyRowsToTensorr=   getrD   r-   r,   r   r/   r   r   )r_   r|   r"   is_trainr   r   r   _add_ops  s    
 

zSparseLookup._add_opsc                 C   s   | j || jdd d S )NTr   )r   rT   )r_   r|   r   r   r   add_train_ops  s    zSparseLookup.add_train_opsc                 C   s~   t  tjddi}| jd | jd  }t|d |d|krB|d nd | jd | jd}|  sj|dkrjd}| j||d	d
 d S )Nr"   r   r   r   r$   )r#   r$   r%   r&   >   r   r   r   r   Fr   )r   r   r'   __name__r@   r>   ro   r   )r_   r|   version_infoZlookup_table_blob_sizer"   r   r   r   add_ops  s"     
zSparseLookup.add_ops)NNr:   NFr;   )T)r   )r   T)r   
__module____qualname__r   r   rZ   rY   rI   rk   rl   ro   rs   rV   r   r   r   r   r   r   r   __classcell__r   r   rc   r   r0   q   s^                             
& 
5I8
r0   )NNNN)!Zcaffe2.python.optimizerr   r   Zcaffe2.python.helpers.arg_scoper   Zcaffe2.pythonr   Zcaffe2.python.layers.layersr   r   r   r	   r
   r   r   r   r   rq   rf   loggingrv   Znumpyr]   rh   	getLoggerr   r   r   r'   r)   r-   r/   r0   r   r   r   r   <module>   s(   ,
    
3