U d) @s ddlmZddlmZmZmZmZmZddlmZddl Z ddl Z ddl Z ddl Z e e dZee jddZdd Zd d Zejd d ZddZedkre\ZZejrdndZedddd edgee!ej"rej#nej$dZ%e&e%eeW5QRXdS)) caffe2_pb2) workspacecoreutilsrnn_cell model_helper) recurrentNZ lstm_benchcCsztd|td}|jgdd|d}|jgdd|d}t|td}||dgdg||d gd gt j d g}t |D]} | t d |d d krtd | || d kr|st j d|d g|ddn|} t j j| t j} | d} || } t j | d | t j}td| td |t||| d | dqtd|||fS)z& Fill a queue with input data z Generating T={} sequence batchesZgenerate_input_initZ inputqueue)Z num_blobscapacityZ labelqueueZgenerate_inputZscratchZ label_scri+ rzGenerating data {}/{}NzFinished data generation)loginfoformatrZNetZCreateBlobsQueuer RunNetOnceZ EnqueueBlobsnprandomseedrangemaxprintrandintZrandZastypefloat32int32FeedBlobProtoappend)TshapeZ num_labels fixed_shapeZgenerate_input_init_netqueue label_queueZgenerate_input_net entry_countstZ random_shapeX batch_sizeLlabelsr'@/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/lstm_benchmark.py generate_datasN     "   r)cCs^tjdd}|jdd\}}|j|d}|j|d}g} |jdkrd} d|jkrz|jsftd |j} t d | t |j D]0} |jd | d | \} } | | | gqtj|||| |j|jg|j d |j|jdd| d \}}}}d|jkr~t dd|j_d|j_n`|jdkrp|jdd} |jjg||dtj||| |j|jd|j d\}}}nds~td|j|d}|j||||gddg\}}|js||g| D]P}|j|||j}|jdkr||j 9}t |t!j"d|j#|gt!j$dq|j%rV|jj&D](}|j'd r,t(j)||j*|j+d!q,||fS)"NZ LSTM_bench)name seq_lengthstargetZ input_datalabel)ownstaticZ static_dagr/z0Random input length is not static RNN compatiblezUsing static RNN of size {}zhidden_init_{}z cell_init_{}Zlstm1T) model input_blobr+initial_statesdim_indim_outscopememory_optimization forward_onlyZ drop_statesZreturn_last_layer_onlyZstatic_rnn_unroll_sizeZdagzUsing DAG net typeZcudnn hidden_init cell_init)rZ cudnnlstm)r0r1r2r3r4r5 num_layersFzUnknown implementationweightssoftmaxlossr ZdtypeZRecurrentNetwork)Z num_threadsZmax_cuda_streams),rZ ModelHelpernetZAddExternalInputsZ DequeueBlobsimplementationrAssertionError seq_lengthrrrr;extendrZLSTM input_dim hidden_dimr6r7rtypeZ num_workersparam_init_netZ ConstantFillZ cudnn_LSTMZ UniformFillZSoftmaxWithLossZFlattenZAddGradientOperatorsZCopyrrrzerosr$r rnn_executorop startswithrZset_rnn_executor_configZrnn_executor_num_threadsZrnn_executor_max_cuda_streams)argsrr Z input_shaper0r+r,r1r&Z init_blobsrir9r:outputZ last_hidden_Z last_stater<r=r>Z init_blobszrKr'r'r( create_modelCs               rRc CsB|j|j}|j|j|jg}t||j||j|j\}}}tdt j |jg|jt j dt ||||\}}t |jt|jt}||j} d} tdt|jj|jrtdt} td| dddtd t}t} td | |jD]t} t|j| | }| |7} t|jj|t}td | | t || | ||| d d |} qtdt |d dt|d d |jrdnd|jr6tdt} td| ddd| d| dkr6t d| d| dt dt|S)Nr+r?rz------ Warming up ------z Memory stats:zGPU memory: {} MBZ max_totaliz ------ Starting benchmark ------r z'Iter: {} / {}. Entries Per Second: {}k.dr z/Done. Total EPS excluding 1st iteration: {}k {}z (with RNN executor)totalz3Max usage differs from current total usage: {} > {}z.This means that costly deallocations occurred.)! data_sizer$rCrEr)rFrrrrarrayrrRrrHZ CreateNetr@timer r ZRunNetrr*gpurZGetGPUMemoryUsageStatsrrZiters_to_reportminsumrJwarning)rMrZinput_blob_shaperr r!r0rO start_timeZ num_itersZ total_itersstatsZ last_time iterationZ iters_onceZnew_timer'r'r( Caffe2LSTMs|         $  r`cCst|S)N)r`)rMr'r'r( BenchmarksracCstjdd}|jdtddd|jdtdd d|jd td d d|jd tddd|jdtddd|jdtddd|jdddd|jdtddd|jdddd|jdddd|jd dd!d|jd"td#d$d|jd%dd&d|jd'tdd(d|jd)tdd*d|S)+NzLSTM benchmark.) descriptionz --hidden_dimi zHidden dimension)rGdefaulthelpz --input_dim(zInput dimensionz --batch_sizezThe batch size.z --seq_lengthzMax sequence lengthz --data_sizei@Bz!Number of data points to generatez--iters_to_reportz&Number of iteration to report progressz--gpu store_truezRun all on GPU)actionrdz--implementationr.z('cudnn', 'own', 'static' or 'static_dag'z --fixed_shapezLWhether to randomize shape of input batches. Static RNN requires fixed shapez--memory_optimizationz+Whether to use memory optimized LSTM or notz--forward_onlyz Whether to run only forward passz --num_layersr zNNumber of LSTM layers. All output dimensions are going to beof hidden_dim sizez--rnn_executorzWhether to use RNN executorz--rnn_executor_num_threadsz*Number of threads used by CPU RNN Executorz--rnn_executor_max_cuda_streamsz:Maximum number of CUDA streams used by RNN executor on GPU)argparseArgumentParser add_argumentintstr)parserr'r'r(GetArgumentParsers rp__main__r Zcaffe2z--caffe2_log_level=0z#--caffe2_print_blob_sizes_at_exit=0z--caffe2_rnn_executor={}z--caffe2_gpu_memory_tracking=1r8)'Z caffe2.protorZ caffe2.pythonrrrrrrrjZnumpyrrXlogging basicConfig getLoggerr setLevelDEBUGr)rRr`debugrarp__name__parse_known_argsrM extra_argsrJZrnn_executor_optZ GlobalInitrZ DeviceOptionrYZ GpuDeviceTypeZCPUZdeviceZ DeviceScoper'r'r'r(sB    ,`F ]