U
    9%e<#                     @   s  U d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZmZ ddlmZmZ dgZ ee! e"d< ej#edddZ$eedddZ%eeedddZ&e!eedddZ'e!eeedddZ(e!ej#edddZ)e!edd d!Z*d"d# Z+d$d% Z,e!eee ee d&d'dZ-eed(d)d*Z.e!eee d+d,d-Z/eedd.d/Z0e!eeee d0d1d2Z1dS )3    )AnyListN)ShardMetadata)ShardedTensor)TensorProperties)DTensor)compute_local_shapecompute_local_offset   )LoadItemTypeSavePlanReadItem	WriteItemWriteItemTypeTensorWriteData)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPES)+_shards_get_overlap_region_wrt_saved_tensor"_check_shard_metadata_pair_overlap create_read_items_for_chunk_list__all__)tensorreturnc                 C   s$   t tdgt|   |  dS )Nr   offsetssizes)r   torchSizelensize)r    r$   k/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor%   s    r&   )shard_mdr   c                 C   s   t t| jt| jdS )Nr   )r   r    r!   shard_offsetsZshard_sizes)r'   r$   r$   r%   _chunk_for_shard,   s    

r)   )sharded_tensorr'   r   c                 C   s   t t||  j|  jdS )Nchunk
propertiesr#   )r   r)   metadataZtensor_propertiesr#   )r*   r'   r$   r$   r%   _sharded_tensor_metadata3   s
    r/   )fqnr   r   c              	   C   sz   |j }|jdkstdtt|j||j}tt|j||j}t	t
| |tjtt||dt| | ddS )Nr
   .Only 1D DeviceMeshes can currently be handled.r   r+   indextypeZtensor_data)device_meshndimAssertionErrorr    r!   r   shape
placementsr	   r   r   r   SHARDr   r   r   create_from_tensorZto_localr#   )r0   r   r5   r   r   r$   r$   r%   _create_write_items_for_dtensor=   s&    r<   )r0   r*   r'   r   c                 C   s(   t |j}tt| |tjt||dS )Nr2   )r    r!   r(   r   r   r   r:   r/   )r0   r*   r'   r   r$   r$   r%   _create_write_item_for_shardU   s    r=   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r   r+   r2   )r    r!   r"   r#   r   r   r   TENSORr   r   r   r;   )r0   r   r   r$   r$   r%   _create_write_item_for_tensor`   s    r?   r0   bytesc                 C   s   t t| tjdS )N)r3   r4   )r   r   r   BYTE_IOr@   r$   r$   r%   _create_write_item_for_bytesiom   s    rC   c              	   C   s.   t tj| t|f|t|ft|fdS N)r4   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rB   r    r!   rE   Zdest_offsetrG   Zstorage_offsetlengthr$   r$   r%   _create_read_item_for_byteiot   s    


rL   c              	   C   s(   t tj| t||t|t|dS rD   )r   r   r>   r    r!   rE   rF   rG   rH   rI   r$   r$   r%   _create_read_item_for_tensor   s    rN   )r0   checkpoint_mdlocal_chunksr   c                 C   s   g }t |D ]\}}t |jD ]\}}t||s2qg }g }	g }
t||dD ]*\}}}}|| |	| |
| qJ|tt| |j||	t| |j|||
d qq|S )aX  
    Creates a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )Zsaved_shardZcurrent_shardrM   )	enumeratechunksr   r   appendrN   r   r   )r0   rO   rP   Z
read_itemsidxshardZstorage_idxZ
storage_mdrH   rF   rI   dimZoffset_for_saved_tensorZoffset_for_current_tensorrK   r$   r$   r%   r      sR      

    )
state_dictr   c                 C   s   g }|   D ]\}}t|tr0|t|| qt|tr^| jD ]}|t||| qDqt|t	j
r||t|| q|t|| qt|S )N)items
isinstancer   rS   r<   r   r.   Zshards_metadatar=   r    Tensorr?   rC   r   )rW   requestsr0   objr'   r$   r$   r%   "_create_default_metadata_only_plan   s    


r]   )r0   objectr   c                    s`   t trt gS t tr8 fdd D S t tjrPt gS t gS d S )Nc                    s   g | ]}t  |jqS r$   )r=   r.   .0rU   r0   r^   r$   r%   
<listcomp>   s   z'_create_write_items.<locals>.<listcomp>)	rY   r   r<   r   local_shardsr    rZ   r?   rC   ra   r$   ra   r%   _create_write_items   s    

rd   c                 C   sP   | j }|jdkstdtt| j|| j}tt| j|| j}t	||dS )Nr
   r1   r   )
r5   r6   r7   r    r!   r   r8   r9   r	   r   )r   r5   r   r   r$   r$   r%   _create_chunk_from_dtensor   s    re   )r0   mdr\   r   c                 C   s   t |tst |tr t|g}nTt |tr>dd | D }n6t |tjrVt|g}nt	d|  ddt
|  t| ||S tt| dt| dddgS d S )Nc                 S   s   g | ]}t |jqS r$   )r)   r.   r_   r$   r$   r%   rb      s     z&_create_read_items.<locals>.<listcomp>z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rJ   )rY   r   r   re   r   rc   r    rZ   r&   
ValueErrorr4   r   rL   r   )r0   rf   r\   rP   r$   r$   r%   _create_read_items   s*    



rh   )2typingr   r   r    Z!torch.distributed._shard.metadatar   Z'torch.distributed._shard.sharded_tensorr   Z0torch.distributed._shard.sharded_tensor.metadatar   Ztorch.distributed._tensorr   Z torch.distributed._tensor._utilsr   r	   Zplannerr   r   r   r   r   r   r.   r   r   r   r   r   r   Z
reshardingr   r   r   str__annotations__rZ   r&   r)   r/   r<   r=   r?   rC   rL   rN   r   r]   rd   re   rh   r$   r$   r$   r%   <module>   sH     	 	 
  =