U
    9%ew                     @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d dlZd dlmZmZ ddlmZ dd	lm Z m!Z!m"Z"m#Z#m$Z$ dd
l%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z. ddl/m0Z0 dgZ1ddddgZ2ee3 e4d< G dd de0Z5e6 Z7dZ8e8d Z9G dd dZ:G dd de:Z;G dd de:Z<G dd de:Z=e Z>ee4d < ej?Z@d!d" ZAeAe_?G d#d$ d$ZBdS )%    N)contextmanagerPath)	AnyBinaryIOCallablecastDictIterableListOptionalUnion)WeakValueDictionary)_get_restore_location_maybe_decode_ascii   )DirectoryReader)_calc___package___normalize_line_endings_normalize_path_resolve_name_sanity_check)demanglePackageManglerPackageUnpickler) _create_directory_from_file_list	Directory)GlobPattern)ImporterPackageImporternumpyz
numpy.coreznumpy.core._multiarray_umathbuiltinsIMPLICIT_IMPORT_ALLOWLISTc                   @   s  e Zd ZU dZeeejf ed< dd fe	ee
jjeef eegef dddZdOed	d
dZeeedddZdPeeeeedddZdQeeedddZdd ZdddddedddZdd  Zd!d" Zeee eed#d$d%Zeed&d'd(Zeed)d*d+Z ed,d-d.Z!d/d0 Z"eeejd1d2d3Z#d4d5 Z$d6d7 Z%dRd9d:Z&d;d<d=d>Z'dSd?d@Z(dAdB Z)dTdCdDZ*e+e dEdFdGdHZ,edIdJdKZ-edLdMdNZ.dS )Ur    a<  Importers allow you to load code written to packages by :class:`PackageExporter`.
    Code is loaded in a hermetic way, using files from the package
    rather than the normal python import system. This allows
    for the packaging of PyTorch model code and data so that it can be run
    on a server or used in the future for transfer learning.

    The importer for packages ensures that code in the module can only be loaded from
    within the package, except for modules explicitly listed as external during export.
    The file ``extern_modules`` in the zip archive lists all the modules that a package externally depends on.
    This prevents "implicit" dependencies where the package runs locally because it is importing
    a locally-installed package, but then fails when the package is copied to another machine.
    modulesc                 C   s   dS )NT )module_namer%   r%   ]/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/package/package_importer.py<lambda>G       zPackageImporter.<lambda>)file_or_buffermodule_allowedc                    sN  t jd   t|t jjr,d _| _nZt|ttfrrt| _t	j
 jsdt j j _qt j _nd _t j| _t jdd j i td _i  _   _ jD ],}||std| d| d	 | q j D ]} | qtj  _ j jd
<   jd< t  _d _d _  fdd _!dS )a  Open ``file_or_buffer`` for importing. This checks that the imported package only requires modules
        allowed by ``module_allowed``

        Args:
            file_or_buffer: a file-like object (has to implement :meth:`read`, :meth:`readline`, :meth:`tell`, and :meth:`seek`),
                a string, or an ``os.PathLike`` object containing a filename.
            module_allowed (Callable[[str], bool], optional): A method to determine if a externally provided module
                should be allowed. Can be used to ensure packages loaded do not depend on modules that the server
                does not support. Defaults to allowing anything.

        Raises:
            ImportError: If the package will use a disallowed module.
        ztorch.package.PackageImporterz<pytorch_file_reader>z<binary>z&torch.package.PackageImporter.metadataserialization_idNz	package 'z' needs the external module 'z%' but that module has been disallowed
__import__Ztorch_package_importerc                     s   t  f| |S Nr   )argskwargsselfr%   r'   r(      r)   z*PackageImporter.__init__.<locals>.<lambda>)"torch_CZ_log_api_usage_once
isinstancePyTorchFileReaderfilename
zip_readerr   strospathisdirr   Z_log_api_usage_metadatar,   _PackageNoderootr$   _read_externextern_modulesImportError_add_externget_all_records	_add_filer"   __dict__copypatched_builtinsr-   r   _manglerstorage_contextlast_map_location	Unpickler)r2   r*   r+   Zextern_modulefnamer%   r1   r'   __init__D   sD    




zPackageImporter.__init__Nnamec                 C   s   | j |}| |S )a  Load a module from the package if it hasn't already been loaded, and then return
        the module. Modules are loaded locally
        to the importer and will appear in ``self.modules`` rather than ``sys.modules``.

        Args:
            name (str): Fully qualified name of the module to load.
            package ([type], optional): Unused, but present to match the signature of importlib.import_module. Defaults to ``None``.

        Returns:
            types.ModuleType: The (possibly already) loaded module.
        )rH   r   _gcd_import)r2   rO   packager%   r%   r'   import_module   s    zPackageImporter.import_module)rQ   resourcereturnc                 C   s   |  ||}| j|S )zLoad raw bytes.

        Args:
            package (str): The name of module package (e.g. ``"my_package.my_subpackage"``).
            resource (str): The unique name for the resource.

        Returns:
            bytes: The loaded data.
        )_zipfile_pathr8   
get_record)r2   rQ   rS   r;   r%   r%   r'   load_binary   s    zPackageImporter.load_binaryutf-8strict)rQ   rS   encodingerrorsrT   c                 C   s   |  ||}|||S )a  Load a string.

        Args:
            package (str): The name of module package (e.g. ``"my_package.my_subpackage"``).
            resource (str): The unique name for the resource.
            encoding (str, optional): Passed to ``decode``. Defaults to ``'utf-8'``.
            errors (str, optional): Passed to ``decode``. Defaults to ``'strict'``.

        Returns:
            str: The loaded text.
        )rW   decode)r2   rQ   rS   rZ   r[   datar%   r%   r'   	load_text   s    zPackageImporter.load_textc           
   	      s    ||}ti i tj fdd  fdd}tj|}	|}||_
tfdd}|  | }	W 5 Q R X tj  |	S )a  Unpickles the resource from the package, loading any modules that are needed to construct the objects
        using :meth:`import_module`.

        Args:
            package (str): The name of module package (e.g. ``"my_package.my_subpackage"``).
            resource (str): The unique name for the resource.
            map_location: Passed to `torch.load` to determine how tensors are mapped to devices. Defaults to ``None``.

        Returns:
            Any: The unpickled object.
        c                    sp   | d} |r&||  }n8jd| || }tjtjjrV	|| | }||| |< d S )Nz.storagez.data/)
Zhas_storageZget_storageZ_typed_storager8   Zget_storage_from_recordr5   r3   r4   r6   Zadd_storage)dtypesizekeylocationrestore_locationrO   storageZtensor)loaded_storagesr2   rI   r%   r'   load_tensor   s    

  z0PackageImporter.load_pickle.<locals>.load_tensorc                    s   t | tstt| d }| dd  }|dkrx|\}}}}|j}|kr\ |||t| | }tjj|j|ddS |dkrt	|dkr|\}	}
|	f|
 S |\}}	}
|kr|	f|
 |< | S d| d	 d S )
Nr   r   rd   T)Zwrap_storager_   	_internalZreduce_package   zVUnknown typename for persistent_load, expected 'storage' or 'reduce_package' but got '')
r5   tupleAssertionErrorr   r_   r3   rd   ZTypedStorageZ_untyped_storagelen)Zsaved_idtypenamer]   Zstorage_typera   rb   r`   r_   rd   funcr/   Z	reduce_id)rf   loaded_reducesre   rc   r2   r%   r'   persistent_load   s:      
z4PackageImporter.load_pickle.<locals>.persistent_loadc                   3   s*   _  _z
d V  W 5 d _ d _X d S r.   )rI   rJ   r%   )map_locationr2   rI   r%   r'   set_deserialization_context  s    
z@PackageImporter.load_pickle.<locals>.set_deserialization_context)rU   r   r3   r4   ZDeserializationStorageContextioBytesIOr8   rV   rK   rp   r   load_utilsZ_validate_loaded_sparse_tensors)
r2   rQ   rS   rq   Zpickle_filerp   Z	data_fileZ	unpicklerrr   resultr%   )rf   ro   re   rq   rc   r2   rI   r'   load_pickle   s     
%


zPackageImporter.load_picklec                 C   s
   | j  S )z
        Returns internal identifier that torch.package uses to distinguish :class:`PackageImporter` instances.
        Looks like::

            <torch_package_0>
        )rH   parent_namer1   r%   r%   r'   id!  s    zPackageImporter.idz**r%   )includeexcluder   )r{   r|   rT   c                C   s   t | j| j ||S )a8  Returns a file structure representation of package's zipfile.

        Args:
            include (Union[List[str], str]): An optional string e.g. ``"my_package.my_subpackage"``, or optional list of strings
                for the names of the files to be included in the zipfile representation. This can also be
                a glob-style pattern, as described in :meth:`PackageExporter.mock`

            exclude (Union[List[str], str]): An optional pattern that excludes files whose name match the pattern.

        Returns:
            :class:`Directory`
        )r   r7   r8   rC   )r2   r{   r|   r%   r%   r'   file_structure*  s       zPackageImporter.file_structurec                 C   s*   d}| j |r&| j |d S dS )aU  Returns the version of python that was used to create this package.

        Note: this function is experimental and not Forward Compatible. The plan is to move this into a lock
        file later on.

        Returns:
            :class:`Optional[str]` a python version e.g. 3.8.9 or None if no version was stored with this package
        z.data/python_versionrX   N)r8   
has_recordrV   r\   strip)r2   Zpython_version_pathr%   r%   r'   python_version=  s
    	
zPackageImporter.python_versionc                 C   s   | j ddjddS )Nz.data/extern_modulesrX   F)keepends)r8   rV   r\   
splitlinesr1   r%   r%   r'   r?   M  s    zPackageImporter._read_extern)rO   r7   
is_packageparentc           
      C   s   |r| j |nd }tjj|| d|d}tj|}|| j|< | j ||_|j	}||d< | |d< ||d< d |d< | j
|d< d|d	< |jtkst|t|j< | ||| |d k	r|d k	st|tjkstt|| | ||}	t|	| |S )
Nz<package_importer>)originr   __spec__
__loader____file__
__cached____builtins__TZ__torch_package__)rH   Zmangle	importlib	machinery
ModuleSpecutilmodule_from_specr$   __name__rE   rG   _package_imported_modulesrk   _install_on_parent	linecachecache	lazycache_compile_sourceexec)
r2   rO   r7   r   r   mangled_filenamespecmodulenscoder%   r%   r'   _make_moduleT  s6    



zPackageImporter._make_module)rO   r   c                 C   s   | j }|dD ]}t|tr(||jkrn|tkrLt| }| j|< |  S t	d| d| j
 d| j |d|j| }t|trt| }| j|< |  S q| ||jt|t|S )N.zNo module named "z" in self-contained archive "zF" and the module is also not in the list of allowed external modules: rN   )r>   splitr5   r=   childrenr#   r   rR   r$   ModuleNotFoundErrorr7   r@   _ExternNoder   source_file)r2   rO   r   curatomr   r%   r%   r'   _load_module}  s    


zPackageImporter._load_module)fullpathr   c                 C   s$   | j |}t|}t||dddS )Nr   T)dont_inherit)r8   rV   r   compile)r2   r   r   sourcer%   r%   r'   r     s    zPackageImporter._compile_source)rT   c                 C   s&   |  t|}| jt|jdS )NrX   )rR   r   r8   rV   r   r\   )r2   r&   r   r%   r%   r'   
get_source  s    zPackageImporter.get_sourcec                 C   s>   z|  |}W n tk
r$   Y d S X |j| k	r4d S t| |S r.   )_get_packagerA   r   _PackageResourceReader)r2   fullnamerQ   r%   r%   r'   get_resource_reader  s    
z#PackageImporter.get_resource_reader)r   rO   r   c                 C   s6   |sd S | j | }|j| kr2t||dd | d S )Nr   rh   )r$   r   setattr
rpartition)r2   r   rO   r   parent_moduler%   r%   r'   r     s
    

z"PackageImporter._install_on_parentc                 C   s$  d }| dd }| dd }|r|| jkr:| | || jkrN| j| S | j| }z
|j}W n tk
r   t|jtjj	r|| j
krtd ||}t||dd t|j|tjstd |||}t||dd ntd ||}t||dd Y nX | ||}| ||| |S )Nr   r   z; {!r} is a c extension module which was not externed. C extension modules                             need to be externed by the PackageExporter in order to be used as we do not support interning them.}.rN   z<; {!r} is a c extension package which does not contain {!r}.z; {!r} is not a package)r   r$   rP   __path__AttributeErrorr5   r   r   r   ExtensionFileLoaderr@   _ERR_MSGformatr   rE   gettypes
ModuleTyper   r   )r2   rO   r;   r   Zmodule_name_no_parentr   msgr   r%   r%   r'   _do_find_and_load  sV    






 
  z!PackageImporter._do_find_and_loadc                 C   s   | j |t}|tkr | |S |d kr@d| d}t||d|dkr\tt|j| j d< n,|dkrtt|j| j d< tt|j	| j d< |S )	Nz
import of z halted; None in sys.modulesrN   r:   zos.pathtypingz	typing.ioz	typing.re)
r$   r   _NEEDS_LOADINGr   r   r   r   r;   rs   re)r2   rO   r   messager%   r%   r'   _find_and_load  s    
zPackageImporter._find_and_loadr   c                 C   s*   t ||| |dkr t|||}| |S )aF  Import and return the module based on its name, the package the call is
        being made from, and the level adjustment.

        This function represents the greatest common denominator of functionality
        between import_module and __import__. This includes setting __package__ if
        the loader did not.

        r   )r   r   r   )r2   rO   rQ   levelr%   r%   r'   rP     s    	zPackageImporter._gcd_importF	recursivec          	      C   s   t |j}t|dr|D ]}t|tsT|r4|d }nd}td| dt|j q|dkr~|st|dr| j||jdd	 qt||s| d
| }z| 	| W q t
k
r } z*|j|kr| j|tdk	rW Y q W 5 d}~X Y qX q|S )zFigure out what __import__ should return.

        The import_ parameter is a callable which takes the name of module to
        import. It is required to decouple the function from assuming importlib's
        import implementation is desired.

        r   z.__all__z``from list''zItem in z must be str, not *__all__Tr   r   N)r   r   hasattrr5   r9   	TypeErrortype_handle_fromlistr   rP   r   rO   r$   r   r   )	r2   r   fromlistr   r&   xwhere	from_nameexcr%   r%   r'   r     s2    




z PackageImporter._handle_fromlistc                 C   s   |dkr|  |}n&|d k	r |ni }t|}|  |||}|s|dkrZ|  |dd S |sb|S t|t|dd  }	t|j}
| j|
d t|
|	   S n| ||S d S )Nr   r   )rP   r   	partitionrl   r   r   r$   r   )r2   rO   globalslocalsr   r   r   globals_rQ   cut_offr&   r%   r%   r'   r-   -  s    
zPackageImporter.__import__c                 C   s^   t |dr0|jjdkr*t|jjdqZ|S n*| |}|jjdkrVt|dn|S dS )zTake a package name or module object and return the module.

        If a name, the module is imported.  If the passed or imported module
        object is not a package, raise an exception.
        r   Nz is not a package)r   r   submodule_search_locationsr   rO   rR   )r2   rQ   r   r%   r%   r'   r   F  s    

zPackageImporter._get_packagec                 C   sZ   |  |}|j| kstt|j}|d k	rHt|}|dd d| S |dd S d S )Nr   /)r   r   rk   r   r   r   replace)r2   rQ   rS   rO   r%   r%   r'   rU   X  s    

zPackageImporter._zipfile_pathz Union[_PackageNode, _ExternNode])atomsrT   c                 C   s   | j }t|D ]\}}|j|d }|d kr>td  }|j|< t|trP|  S t|tr|d|d | }t	d| dt|tst
|}q|S )Nr   z&inconsistent module structure. module z% is not a package, but has submodules)r>   	enumerater   r   r=   r5   r   _ModuleNodejoinrA   rk   )r2   r   r   ir   noderO   r%   r%   r'   _get_or_create_packageb  s    


z&PackageImporter._get_or_create_package)r7   c                 C   s   | d^ }}t|dkr,|d dkr,dS | |}t|trPtd| d|dkr`||_n*|d	r|dtd	  }t||j	|< dS )
zAssembles a Python module out of the given file. Will ignore files in the .data directory.

        Args:
            filename (str): the name of the file inside of the package archive to be added
        r   r   r   z.dataNz>inconsistent module structure. package contains a module file z2 that is a subpackage of a module marked external.z__init__.pyz.py)
r   rl   r   r5   r   rA   r   endswithr   r   )r2   r7   prefixlastrQ   package_namer%   r%   r'   rD   u  s    



zPackageImporter._add_file)extern_namec                 C   s8   | d^ }}| |}t|tr(d S t |j|< d S )Nr   )r   r   r5   r   r   )r2   r   r   r   rQ   r%   r%   r'   rB     s
    

zPackageImporter._add_extern)N)rX   rY   )N)Nr   )NNr%   r   )N)/r   
__module____qualname____doc__r	   r9   r   r   __annotations__r   r3   r4   r6   r   r   r   boolrM   rR   bytesrW   r^   r   rx   rz   r   r}   r   r?   r   r   r   r   r   r   r   r   r   rP   r   r-   r   rU   r   r   rD   rB   r%   r%   r%   r'   r    0   sf   
D  _
     )		2
(

zNo module named z{!r}c                   @   s   e Zd ZdS )	_PathNodeNr   r   r   r%   r%   r%   r'   r     s   r   c                   @   s   e Zd Zee dddZdS )r=   r   c                 C   s   || _ i | _d S r.   )r   r   r2   r   r%   r%   r'   rM     s    z_PackageNode.__init__N)r   r   r   r   r9   rM   r%   r%   r%   r'   r=     s   r=   c                   @   s    e Zd ZdgZedddZdS )r   r   r   c                 C   s
   || _ d S r.   r   r   r%   r%   r'   rM     s    z_ModuleNode.__init__N)r   r   r   	__slots__r9   rM   r%   r%   r%   r'   r     s   r   c                   @   s   e Zd ZdS )r   Nr   r%   r%   r%   r'   r     s   r   r   c                 C   s(   t | r | jtkr t| j jS t| S r.   )inspectisclassr   r   r   _orig_getfile)objectr%   r%   r'   _patched_getfile  s    

r   c                   @   s8   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d ZdS )r   zPrivate class used to support PackageImporter.get_resource_reader().

    Confirms to the importlib.abc.ResourceReader interface. Allowed to access
    the innards of PackageImporter.
    c                 C   s   || _ || _d S r.   )importerr   )r2   r   r   r%   r%   r'   rM     s    z_PackageResourceReader.__init__c                 C   s    ddl m} || j| j|S )Nr   )rt   )rs   rt   r   rW   r   )r2   rS   rt   r%   r%   r'   open_resource  s    z$_PackageResourceReader.open_resourcec                 C   sH   t | jjtr@| jjtj| j|r@tj| jjj	| j|S t
d S r.   )r5   r   r8   r   r~   r:   r;   r   r   	directoryFileNotFoundError)r2   rS   r%   r%   r'   resource_path  s       z$_PackageResourceReader.resource_pathc                 C   s   | j | j|}| j j|S r.   )r   rU   r   r8   r~   )r2   rO   r;   r%   r%   r'   is_resource  s    z"_PackageResourceReader.is_resourcec              	   c   s   ddl m} | jdd}|| j| j}| jj }t }|D ]f}z||	|}W n t
k
rp   Y qBY nX |jj}t|dkr|jV  qB||krB|| |V  qBd S )Nr   r   r   r   )pathlibr   r   r   r   rU   r8   rC   setrelative_to
ValueErrorr   rO   rl   add)r2   r   r7   fullname_pathfilessubdirs_seenrelativery   r%   r%   r'   contents  s     


z_PackageResourceReader.contentsN)	r   r   r   r   rM   r   r   r   r  r%   r%   r%   r'   r     s   r   )Cr"   r   Zimportlib.machineryr   rs   r   os.pathr:   r   
contextlibr   r   r   r   r   r   r   r   r	   r
   r   r   r   weakrefr   r3   Ztorch.serializationr   r   Z_directory_readerr   Z
_importlibr   r   r   r   r   Z	_manglingr   r   Z_package_unpicklerr   Zfile_structure_representationr   r   Z
glob_groupr   r   r   r   r#   r9   r   r    r   r   _ERR_MSG_PREFIXr   r   r=   r   r   r   getfiler   r   r   r%   r%   r%   r'   <module>   sX    ,
    f