U
    *-e9                     @   s  U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZ d dlmZmZmZm Z m!Z!m"Z"m#Z# d d	l$m%Z%m&Z&m'Z' d d
l(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 e3e4Z5ej6e7d< ddddddgZ8G dd deZ9G dd deZ:ee;ef e!edddZ<e	e e	e dddZ=ee;ef e>edddZ?d-e	e e>e
e	e e!f dd dZ@e"e!d!d"d#ZAeee>d$d%d&ZBejCee>d'd(d)ZDe	e e!e>d*d+d,ZEdS ).    N)ChainMap)reduce)ListTupleDictAnyUnioncast)narrow_tensor_by_index)DTensor)SavePlannerLoadPlannerSavePlanLoadPlanReadItem	WriteItemWriteItemType)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadataMetadataIndexMetadataSTATE_DICT_TYPESTORAGE_TYPES)_create_read_items_create_write_items"_create_default_metadata_only_plan)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors)dedup_tensors)find_state_dict_object)set_elementloggerDefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc                   @   s   e Zd ZU eed< deeeddddZeedddd	Ze	d
ddZ
ee	 eee	 ef dddZe	e	dddZeeejejf dddZeedddZeedddZdS )r$   mappingsTN)r   flatten_sharded_tensorsdedup_replicated_tensorsreturnc                 C   s   || _ || _|| _i | _d S N)r   r+   r,   r*   )selfr   r+   r,    r0   m/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/torch/distributed/checkpoint/default_planner.py__init__G   s    zDefaultSavePlanner.__init__
state_dictis_coordinatorr-   c                 C   s2   | j rt |\}| _| jr"t|}|| _|| _d S r.   )r   r*   r+   r   r4   r5   )r/   r4   r5   r0   r0   r1   set_up_plannerR   s    z!DefaultSavePlanner.set_up_plannerr-   c                 C   s0   t | j| j}| jr$tj|| jd}|| _| jS )NZplanner_data)r(   r4   r5   r   dataclassesreplacer*   plan)r/   r;   r0   r0   r1   create_local_plan\   s     z$DefaultSavePlanner.create_local_plan	all_plansr-   c                 C   sr   | j rt|}t|\}}| jrHdd |D }tt| }tj||d}t||sZt	d|| _
|| _| j
| jfS )Nc                 S   s   g | ]
}|j qS r0   r8   ).0pr0   r0   r1   
<listcomp>s   s     z9DefaultSavePlanner.create_global_plan.<locals>.<listcomp>r8   zFailed to validate global plan)r,   r    r)   r   dictr   r9   r:   _validate_global_plan
ValueErrorglobal_planmetadata)r/   r>   rE   rF   Zplanner_data_dictZmerged_mappingsr0   r0   r1   create_global_planf   s     
z%DefaultSavePlanner.create_global_plannew_planr-   c                 C   s
   || _ |S r.   )r;   r/   rI   r0   r0   r1   finish_plan   s    zDefaultSavePlanner.finish_plan)
write_itemr-   c                 C   s   |  |j}| ||S r.   )lookup_objectindextransform_object)r/   rL   objectr0   r0   r1   resolve_data   s    zDefaultSavePlanner.resolve_datarN   r-   c                 C   s   t | j|S zo
        This is an extension from the planner interface to make it easy to extend the default planner
        r!   r4   r/   rN   r0   r0   r1   rM      s    z DefaultSavePlanner.lookup_object)rL   rP   c                 C   s(   |j tjkr$t }t|| |}|S rS   )typer   BYTE_IOioBytesIOtorchsave)r/   rL   rP   bytesr0   r0   r1   rO      s
    z#DefaultSavePlanner.transform_object)TTT)__name__
__module____qualname__r   __annotations__boolr2   r   r6   r   r<   r   r   r   rG   rK   r   r   rZ   TensorrX   rY   rQ   r   r   rM   rO   r0   r0   r0   r1   r$   D   s0   
    
c                   @   s   e Zd ZU dZeed< eed< d$eeddddZee	edd	d
dZ
edddZee ee dddZeedddZeejddddZedddZeejddddZeejddd Zeejd!d"d#ZdS )%r%   z
    DefaultLoadPlanner that adds multiple features on top of LoadPlanner.

    In particular it adds the following:

    flatten_state_dict: Handle state_dict with nested dicts
    flatten_sharded_tensors: For FSDP in 2D parallel mode
    original_state_dictr*   TN)r   r+   r-   c                 C   s   || _ || _i | _i | _d S r.   )r   r+   rc   r*   )r/   r   r+   r0   r0   r1   r2      s    zDefaultLoadPlanner.__init__)r4   rF   r5   r-   c                 C   s>   || _ | jrt|}| jr(t|\}| _|| _|| _|| _d S r.   )rc   r+   r   r   r*   r4   rF   r5   )r/   r4   rF   r5   r0   r0   r1   r6      s    z!DefaultLoadPlanner.set_up_plannerr7   c                 C   s   t | j| jS r.   )r&   r4   rF   )r/   r0   r0   r1   r<      s    z$DefaultLoadPlanner.create_local_plan)rE   r-   c                 C   s   t |S r.   )r'   )r/   rE   r0   r0   r1   rG      s    z%DefaultLoadPlanner.create_global_planrH   c                 C   s   |S r.   r0   rJ   r0   r0   r1   rK      s    zDefaultLoadPlanner.finish_plan)	read_itemvaluer-   c                 C   s>   | j r&t| j| j|jj t| nt|| j|jj< d S r.   )	r   r"   rc   r*   
dest_indexfqnrZ   loadr4   )r/   rd   re   r0   r0   r1   
load_bytes   s    zDefaultLoadPlanner.load_bytes)rd   c                 C   s   |  |j}| ||S r.   )lookup_tensorrf   transform_tensorr/   rd   tensorr0   r0   r1   resolve_tensor   s    z!DefaultLoadPlanner.resolve_tensor)rd   rm   r-   c                 C   s   d S r.   r0   rl   r0   r0   r1   commit_tensor   s    z DefaultLoadPlanner.commit_tensorrR   c                 C   s   t | j|S rS   rT   rU   r0   r0   r1   rj      s    z DefaultLoadPlanner.lookup_tensor)rd   rm   c                 C   s   t ||j|jS rS   )r
   Zdest_offsetslengthsrl   r0   r0   r1   rk      s
      z#DefaultLoadPlanner.transform_tensor)TT)r]   r^   r_   __doc__r   r`   r   ra   r2   r   r6   r   r<   r   rG   rK   r   rX   rY   ri   rn   rZ   rb   ro   r   rj   rk   r0   r0   r0   r1   r%      s.   
	  
)r4   rF   r-   c                 C   sb   g }|   D ]L\}}|j| }t|trH|j d k	rX|t|||7 }q|t|||7 }qt|S r.   )itemsstate_dict_metadata
isinstancer   device_meshget_coordinater   r   )r4   rF   requestsrg   objmdr0   r0   r1   r&      s    

r=   c                 C   s   | S )z
    Create global load plan used by DefaultLoadPlanner.

    The default load behavior involved no global coordination and this function
    currently doesn't change the local plans.
    r0   )r>   r0   r0   r1   r'   
  s    	r3   c                 C   sd   g }|   D ]N\}}t|tr<|j dk	rZ|t||7 }qt|tjsL|r|t||7 }qt|S )a  
    Create the ``SavePlan`` used by DefaultSavePlanner.

    On non-coordinator ranks, this function ignores tensors and non-tensor objects,
    only producing writes for ShardedTensor objects.

    On the coordinator rank, produce writes for all values.
    N)	rr   rt   r   ru   rv   r   rZ   rb   r   )r4   r5   rw   rg   rx   r0   r0   r1   r(     s    
T)r>   rewrite_index_hintsr-   c           
      C   s"  i }g }| D ]}g }|j D ]}|jtjks<|jj|ks<t|jtjkrbt ||jj< |	| q|j
dk	spttt||jjt|j
j|j
jg d}|}|rtj|jt|jd}	tj||	d}|	| |j
jdk	std|jj d|j	|j
j q|	tj||d q|t|fS )a6  
    Create the global plan and metadata used by DefaultSavePlanner.

    Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans.

    The only global planning change is to update index hints in all ``MetadataIndex`` objects if
    ``rewrite_index_hints`` is True.
    N)
propertiessizechunks)rN   zZ
                    Cannot create MD for tensor without bounds.
                    FQN: z
                )rr   )rr   rV   r   ZSHARDrN   rg   AssertionErrorrW   r   appendZtensor_datar	   r   
setdefaultr{   r|   r9   r:   lenr}   chunkr   )
r>   rz   ry   Z	new_plansr;   Z	new_itemsitemZ	tensor_mdZnew_itemZ	new_indexr0   r0   r1   r)   .  sN    

 

)r4   r-   c                 C   s   t | }t|g\}}|S )z^
    Return the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.
    )r   r)   )r4   r;   _ry   r0   r0   r1   _create_default_local_metadatae  s    r   )box0box1r-   c                 C   sd   t | j}t|D ]L}| j| |j| |j|  kr: dS |j| | j| | j|  kr dS qdS )zC
    Checks if two boxes overlap. Tuples are (offset, lengths)
    FT)r   offsetsrangesizes)r   r   Zndimsir0   r0   r1   _check_box_overlapn  s    
r   )outer_box_size	inner_boxr-   c                 C   s`   t t| D ]N}|j| dk r$ dS |j| dk r8 dS |j| |j|  | | kr dS qdS )Nr   FT)r   r   r   r   )r   r   r   r0   r0   r1   _check_box_bounds  s    r   )rE   rF   r-   c           
   	   C   s   d}|j  D ]\}}t|tr"qt|jdkr2qd}t|jD ]t\}}t|j|sjt	
d||j| d}|ttj|jd7 }|j|d d  D ]"}t||rt	
d||| d}qq@ttj|jd}	||	krt	
d||	| d}q|S )NTr   z~
                        key:%s has out of bounds chunk:
                        tensor-size:%s chunk: %s
                    F   z$key:%s has overlapping chunks: %s %szq
                    key:%s invalid fill tensor-volume:
                    %s chunks-volume: %s
                )rs   rr   rt   r   r   r|   	enumerater}   r   r#   warningr   operatormulr   r   )
rE   rF   all_goodkeyre   Zchunks_volumeZ	chunk_idxZchunk0Zchunk1Ztensor_volumer0   r0   r1   rC     sH    
  
     rC   )T)Fr9   rX   loggingr   collectionsr   	functoolsr   typingr   r   r   r   r   r	   rZ   Ztorch.distributed._shard._utilsr
   Ztorch.distributed._tensorr   Z$torch.distributed.checkpoint.plannerr   r   r   r   r   r   r   Z%torch.distributed.checkpoint.metadatar   r   r   r   r   r   r   Z,torch.distributed.checkpoint.planner_helpersr   r   r   Z)torch.distributed.checkpoint._nested_dictr   r   Z2torch.distributed.checkpoint._sharded_tensor_utilsr   Z+torch.distributed.checkpoint._dedup_tensorsr    Z"torch.distributed.checkpoint.utilsr!   Z&torch.distributed.checkpoint._traverser"   	getLogger__file__r#   Loggerr`   __all__r$   r%   strr&   r'   ra   r(   r)   r   r   Sizer   rC   r0   r0   r0   r1   <module>   st     $
$
XS

  7
   