U
    0-e^                     @   s\  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZmZmZmZmZmZmZmZ e rd d
lmZ nd dlmZ dd Zdd Z dd Z!dd Z"dd Z#dd Z$dd Z%dd Z&dd Z'dd Z(d d! Z)d"d# Z*d$d% Z+d&d' Z,d(d) Z-d*d+ Z.e/d,krXe.  dS )-    N)deepcopy)Path)
DataLoader)Accelerator)prepare_data_loader)AcceleratorState)RegressionDatasetare_the_same_tensors)DistributedTypegatheris_bf16_availableis_ipex_availableis_npu_availableis_xpu_availableset_seedsynchronize_rng_states)RegressionModel4XPU)RegressionModelc                 C   s   t d| j  d S )NzPrinting from the main process printprocess_indexstate r   j/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/accelerate/test_utils/scripts/test_script.py
print_main2   s    r   c                 C   s   t d| j  d S )Nz%Printing from the local main process )r   local_process_indexr   r   r   r   print_local_main6   s    r   c                 C   s   t d| j  d S )NPrinting from the last process r   r   r   r   r   
print_last:   s    r   c                 C   s   t d| d| j  d S )NPrinting from process : r   )r   process_idxr   r   r   print_on>   s    r#   c               
   C   sr  t  } | j}td}|  X | jrPtd t|d}|d W 5 Q R X n t|d}|d W 5 Q R X W 5 Q R X | 	  | jr6t|d}d
| }W 5 Q R X zf|dstd|d	kr|dstd|d| jd	 kstd
|dd	  d| j W n  tk
r4   |   Y nX | jrP| rP|  | 	  t }t| | t| j W 5 Q R X |  }| jr|dkst| dn |  dkst| d|d |d t| | t| j W 5 Q R X | jr.|  dksDtn|  dksDt|d |d t| |  t!| j W 5 Q R X | j"r|  d| jjd	  kstn|  dkst|d |d t#|D ]}t| | j$t%|d| j| W 5 Q R X | j&|kr@|  d| d| j& ksVtn|  dksVt|d |d qd S )Nzcheck_main_process_first.txt皙?za+zCurrently in the main process
zNow on another process
r zMain process was not first   zOnly wrote to file zNow on another processz times, not z Printing from the main process 0z$ != Printing from the main process 0z != ""r   z&Printing from the local main process 0r   )r   r    r!   )'r   num_processesr   Zmain_process_firstis_main_processtimesleepopenwritewait_for_everyonejoin	readlines
startswithAssertionErrorendswithcountunlinkexistsioStringIO
contextlibredirect_stdoutZon_main_processr   r   getvaluerstriptruncateseekZon_local_main_processr   Zis_local_main_processZon_last_processr   is_last_processrangeZ
on_processr#   r   )acceleratorr(   pathftextresultr"   r   r   r   process_execution_checkB   sv    

 



&

 &
rF   c                  C   s$   t  } | jdkrtd t|  d S )Nr   zTesting, testing. 1, 2, 3.)r   r   r   r   r   r   r   init_state_check   s    
rG   c                  C   s   t  } tdg tt s$td| jtjkrRtdg ttj	 s~tdn,| jtj
kr~tdg ttj s~tdt }tdg|d t| std	| jd
krtd d S )Ntorchz*RNG states improperly synchronized on CPU.cudaz*RNG states improperly synchronized on GPU.xpuz*RNG states improperly synchronized on XPU.	generator)rK   z0RNG states improperly synchronized in generator.r   zAll rng are properly synched.)r   r   r	   rH   Zget_rng_stater2   distributed_typer
   	MULTI_GPUrI   Z	MULTI_XPUrJ   	GeneratorZ	get_stater   r   )r   rK   r   r   r   rng_sync_check   s    



rO   c                  C   s(  t  } d| j }tt|dd}t|| j| j| jdd}g }|D ]}|t| q@t	
|}t| j|t| t	| t	d| stdtt|dd}t|| j| j| jddd}g }|D ]}|t| qt	
|}t	| t	d| s
td| jdkrtd	 tt|ddd
}t|| j| j| jdd}g }|D ]}|t| qPt	
| }|  |tt|kstdtt|ddd
}t|| j| j| jddd}g }|D ]}|t| qt	
| }|  |tt|kstd| jdkr$td d S )N       
batch_sizeT)put_on_devicer   %Wrong non-shuffled dataloader result.)rT   split_batchesz Non-shuffled dataloader passing.rS   shuffle!Wrong shuffled dataloader result.zShuffled dataloader passing.)r   r(   r   r@   r   devicer   appendr   rH   catr   typeequalcpuarangelongr2   tolistsortlistr   r   lengthdlrE   batchr   r   r   dl_preparation_check   sf    

$
&ri   c               	   C   s  t  } d| j }tt|dd}t|| j| j| jddd}g }|D ]}|t| qBt	
|}t	| t	d| stdtt|dd}t|| j| j| jdddd}g }|D ]}|t| qt	
|}t	| t	d| std| jdkrtd	 tt|ddd
}t|| j| j| jddd}g }|D ]}|t| qBt	
| }|  |tt|kstdtt|ddd
}t|| j| j| jdddd}g }|D ]}|t| qt	
| }|  |tt|kstd| jdkrtd d S )NrP   rQ   rR   T)rT   dispatch_batchesr   rU   )rT   rV   rj   z(Non-shuffled central dataloader passing.rW   rY   z$Shuffled central dataloader passing.)r   r(   r   r@   r   rZ   r   r[   r   rH   r\   r^   r_   r`   ra   r2   r   rb   rc   rd   r   re   r   r   r   central_dl_preparation_check   s    
     
$	
$     	rk   c                 C   s   t d |d t| dd}t||d|d}t }tjj| dd}t	dD ]F}|D ]<}|
  ||d }	tjj|	|d	 }
|
  |  qXqP||fS )
N*   )rf   seedTrS   rX   rK   r$   lr   xy)r   manual_seedr   r   r   rH   optimSGD
parametersr@   	zero_gradnn
functionalmse_lossbackwardstep)rf   rS   rK   	train_settrain_dlmodel	optimizerepochrh   outputlossr   r   r   mock_training#  s    
r   c                  C   s  t  } t }d}|d | j }t||| j |\}}t|jsHtdt|jsZtdt	 }t
||d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]H}
|D ]>}|  ||d
 }tjj||d }|| |	  qq|| }t|j|js*tdt|j|jsDtd|d t	dd}t
||| j d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]L}|D ]@}|  ||d
 }tjj||d }|| |	  qq|| }t|j|js,tdt|j|jsFtd|d tj sdt rftd t    t	dd}t
||d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]L}|D ]@}|  ||d
 }tjj||d }|| |	  qސq|| }t|j|jsLtdt|j|jsftdtj rtd t    t	dd}tj!dd}||}|j|dd}t"ddgj#tj$|j%d}||}t& rtd t    t	dd}t
||d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]L}|D ]@}|  ||d
 }tjj||d }|| |	  qXqP|| }t|j|jstdt|j|jstdt' rtd t    t	ddd}t
||d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]L}|D ]@}|  ||d
 }tjj||d }|| |	  qdq\|| }t|j|jstdt|j|jstdt( rtd t    t	ddd}t
||d|d}t }tjj| dd}	||||	\}}}	td |d td	D ]L}|D ]@}|  ||d
 }tjj||d }|| |	  qpqh|| }t|j|jstdt|j|jstdd S )NrQ      z0Did not obtain the same model on both processes.Trn   r$   ro   rl   rq   rr   rs   z=Did not obtain the same model on CPU or distributed training.zVTraining yielded the same results on one CPU or distributed setup with no batch split.)rV   zSTraining yielded the same results on one CPU or distributes setup with batch split.zFP16 training check.Zfp16)mixed_precisionzKeep fp32 wrapper check.   )Zkeep_fp32_wrapperr'   )ZdtyperZ   zBF16 training check.Zbf16zipex BF16 training check.)r   r_   zxpu BF16 training check.Fz=Did not obtain the same model on XPU or distributed training.))r   rH   rN   r(   r   r	   ar2   br   r   r   ru   rv   rw   preparer   rt   r@   rx   ry   rz   r{   r|   r}   Zunwrap_modelr_   allcloser   rI   Zis_availabler   Z_reset_stateZLinearZTensortoZfloat16rZ   r   r   r   )r   rK   rS   rf   r~   Z	old_modelrA   r   r   r   r   rh   r   r   _Zmodel_with_fp32_wrapperZinput_tensorr   r   r   training_check4  s   


















r   c               	   C   s   t  } ttdd| j }| |,}t|dksLtd| j dt| W 5 Q R X ttdd| j d }| j|ddF}| jrt	
t|| j }t||kstd	| j dt| W 5 Q R X |   d S )
Nr   r   z4Each process did not have two items. Process index: z
; Length: rq   r'   T)Zapply_paddingz;Last process did not get the extra item(s). Process index: )r   rd   r@   r(   split_between_processeslenr2   r   r?   mathceilr.   )r   dataresultsZnum_samples_per_devicer   r   r   !test_split_between_processes_list  s     

r   c               
   C   s  t  } ddddddddg}d	d
ddddddg}tddddddddg}| jdkr|||d}t|}| |$}| jdkr|d	 |d	 d d| j  kstnn| jdkr|d	 |d	 dd  kstnF| jdkr|d	 |d	 dd  kstd|d	 d  d|d	  | jdkrD|d
 |d
 d d| j  kstnV| jdkrp|d
 |d
 dd  kstn*| jdkr|d
 |d
 dd  kst| jdkrt|d |d d d| j  std|d d d| j   d|d  n| jdkrFt|d |d dd  std|d dd   d|d  nN| jdkrt|d |d dd  std|d dd   d|d  W 5 Q R X | 	  d S )Nr'   r   rq   r            rQ   r   r   cderC   gh)r'   r   r   )r   r   r   r   z
Expected: z
, Actual: z7Did not obtain expected values on process 0, expected `z`, received: z7Did not obtain expected values on process 2, expected `z7Did not obtain expected values on process 4, expected `)
r   rH   tensorr(   r   r   r   r2   r   r.   )r   r   r   r   r   Z	data_copyr   r   r   r   (test_split_between_processes_nested_dict  sT    
$
:&  $  r   c               
   C   s   t  } | jdkrtddddgddddgg| j}| |^}| jdkrtt|tddddg| jst	n&t|tddddg| jst	W 5 Q R X | 
  d S )	Nr'   r   r   rq   r   r   r   r   )r   r(   rH   r   r   rZ   r   r   r   r2   r.   )r   r   r   r   r   r   #test_split_between_processes_tensor  s    
&
(0r   c                  C   sH   t  } |  dkst| jr$|   |  dks4t|  dksDtd S )NFT)r   Zcheck_triggerr2   r)   Zset_trigger)rA   r   r   r   test_trigger   s    r   c                  C   s@  t  } | j}|jdkrtd t  |  |jtjkrDt	j
 }n|j}||jkr|jdkrftd t  |jdkr~td t  |jdkrtd t  |jdkrtd t  |jdkrtd t  |jdkrtd t  |jtjkrt  |jtjkrd S |jdkrtd	 t  |jdkr6td
 t  d S )Nr   z**Initialization**z
**Test process execution**z+
**Test split between processes as a list**z+
**Test split between processes as a dict**z-
**Test split between processes as a tensor**z1
**Test random number generator synchronization**z 
**DataLoader integration test**z
**Training integration test**z
**Breakpoint trigger test**)r   r   r   r   rG   r.   rL   r
   rM   rH   rI   Zdevice_countr(   r   rF   r   r   r   rO   ri   ZTPUrk   Z	DEEPSPEEDr   r   )rA   r   Znum_processes_per_noder   r   r   main1  sL    







r   __main__)0r9   r7   r   r*   copyr   pathlibr   rH   Ztorch.utils.datar   Z
accelerater   Zaccelerate.data_loaderr   Zaccelerate.stater   Zaccelerate.test_utilsr   r	   Zaccelerate.utilsr
   r   r   r   r   r   r   r   r   r   r   r   r   r#   rF   rG   rO   ri   rk   r   r   r   r   r   r   r   __name__r   r   r   r   <module>   sD   (I=A (&6
