U -e@sUddlZddlmZddlmZmZmZddlZddl Z ddl m Z ddl mZddlmZdd lmZer~ddlZddlZeZdaeeed <Gd d d eed efZdS)N)Mapping) TYPE_CHECKINGDictOptional)config) get_logger) map_nested)TensorFormatterDEVICE_MAPPINGcseZdZdfdd ZeeedfdddZdd Zd d Z d d Z e dddZ e jedddZe jddddZe jedddZZS) JaxFormatterNc stj|dddl}ddlm}t||rFtd|dt|dt|trT|nt| d|_ t dkrv| a |j t t krtd|j dt t d t| dd t| d|_ ||_dS) N)featuresr)Devicez Expected z to be a `str` not z, as `jaxlib.xla_extension.Device` is not serializable neither with `pickle` nor with `dill`. Instead you can surround the device with `str()` to get its string identifier that will be internally mapped to the actual `jaxlib.xla_extension.Device`.zDevice with string identifier z) not listed among the available devices: z), so falling back to the default device: .)super__init__jaxZjaxlib.xla_clientr isinstance ValueErrortypestrdevicesdevicer _map_devices_to_strlistkeysloggerwarningjnp_array_kwargs)selfrrrrr __class__b/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/formatting/jax_formatter.pyr's    ,zJaxFormatter.__init__zjaxlib.xla_extension.Device)returncCsddl}dd|DS)NrcSsi|]}t||qSr#)r).0rr#r#r$ Fsz4JaxFormatter._map_devices_to_str..)rr)rr#r#r$rBsz JaxFormatter._map_devices_to_strcsLddlddlm}ttrHrHtfddDrH|jddSS)Nrc3s:|]2}t|jo0|jdjko0|jdjkVqdS)rN)rArrayshapedtype)r&xcolumnrr#r$ Msz,JaxFormatter._consolidate..)Zaxis)r jax.numpynumpyrrallstack)r r-jnpr#r,r$ _consolidateHs zJaxFormatter._consolidatec CsHddl}ddlm}t|tttdfr,|St|tjtj frVt |j tjrV| Si}t|tj tj frt |j tjr|jjrd|ji}qd|ji}n^t|tj tj frt |j tjrd|ji}n0tjrdtjkrddl}t||jjrt|}tdkr |a|t|j"|j|f||j W5QRSQRXdS)Nrr*PIL)!rr/r0rrbytesrnp characterndarrayZ issubdtyper*tolistnumberintegerrZjax_enable_x64Zint64Zint32ZfloatingZfloat32Z PIL_AVAILABLEsysmodulesZ PIL.ImageZImageZasarrayr rZdefault_devicerarrayr)r valuerr3Z default_dtyper5r#r#r$ _tensorizeSs* ""  "   zJaxFormatter._tensorizecsddl}tjrFdtjkrFddl}t||jrF| dSt |drdt||j sd|}t|tjr|jtkrfdd|DSn&t|ttfrfdd|DS|S)Nrtorchr# __array__csg|]}|qSr#recursive_tensorizer&Z substructr r#r$ sz5JaxFormatter._recursive_tensorize..csg|]}|qSr#rDrFrGr#r$rHs)rrZTORCH_AVAILABLEr=r>rBrZTensorrAdetachcpur0hasattrr(rCr7r9r*objectr4rtuple)r data_structrrBr#rGr$_recursive_tensorizexs   z!JaxFormatter._recursive_tensorize)rNcCst|j|ddS)NF)Zmap_list)r rO)r rNr#r#r$rEsz JaxFormatter.recursive_tensorize)pa_tabler%cCs$||}|j|}||SN)numpy_arrow_extractorZ extract_rowpython_features_decoderZ decode_rowrE)r rProwr#r#r$ format_rows zJaxFormatter.format_row jax.ArraycCs:||}|j||jd}||}||}|S)Nr)rRZextract_columnrSZ decode_columnZ column_namesrEr4)r rPr-r#r#r$ format_columns   zJaxFormatter.format_columncCsD||}|j|}||}|D]}|||||<q(|SrQ)rRZ extract_batchrSZ decode_batchrEr4)r rPbatchZ column_namer#r#r$ format_batchs   zJaxFormatter.format_batch)NN)__name__ __module__ __qualname__r staticmethodrrrr4rArOdictrEpaTablerrUrWrY __classcell__r#r#r!r$r &s %r rV)r=collections.abcrtypingrrrr0r7Zpyarrowr_rZ utils.loggingrZutils.py_utilsr Z formattingr rZjaxlibrr r^__annotations__r r#r#r#r$s