U sVcy@sP UddlmZddlZddlmZddlmZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZmZddlZddlZddlmZmZmZddlmZdd lmZdd lmZmZm Z m!Z!m"Z"ddl#Z$dd l#m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dd l/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5dd l6m7Z7m8Z8m9Z9ddl:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTddlUmVZVmWZWddlXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`ddlambZbmcZcmdZdmeZeddlfmgZgmhZhmiZiddljmkZkddllmmZme r@ddl#mnZnmoZodZpdZqddddgZrdesd<dd d!d"gZtdesd#<eud$d%d&d'gZvdesd(<d)d*d+d,gZwdesd-<erevZxetewZyezd.d/gZ{desd0<d1d2gZ|desd3<e}d4d5gZ~desd6<ed7d8gZdesd9<d:d;gZdesd<<d=d>gZdesd?<ed@gZdesdA<edBgZdesdC<edDgZdesdE<e{exZee~eeeeeeZejejejejejejejejgZdFdGdHe Zdeje$jezdIe$jedJgZdKdLejejfDZesddlZeeeegZeeeegZeeZeegZe egZedMedNedOedPgZeegZdQdLdRDZdSdLdRDZegZeeeeeeeZedTZeefZdUdVdWdXZdUdVdYdZZedUdVd[d\Zd@dVd]d^Zdd`daZdbdcZd7dVdddeZd dgdVdhdiZd!dkdVdldmZd"dndVdodpZd#dgdVdqdrZd$dsdVdtduZd%dvdVdwdxZÐd&dydVdzd{ZĐd'd|dVd}d~ZŐd(ddVddZƐd)dddddZǐd*dddddZȐd+dddddZɐd,ddZddZːd-dddddZdejfddVddZ͐d.ddVddZΐd/ddVddZϐd0ddVddZddVddZѐd1ddVddZҐd2ddVddZӐd3ddVddZԐd4ddVddZՐd5ddVddZddVddZddZddVddZِd6ddVddZڐd7ddgdddZېd8ddVddZܐd9ddZݐd:ddVdd„ZGddĄde-ZGddƄde'ZGddȄde%Zd;ddʄZdd7d̜dd΄ZdddќddӄZe$jjjZddՄZd7ddלddلZddۄZdd݄Zdd߄ZddZddZddZd@dVddZddddddddddddddddddddddddddddddAdadCddd6dd<ddddd d d^dd3d0dedՐd dېd dd dddddddddrdmdddddddpdxddddudddddd~diddddd{ddddddEdddd\dZdddddݐddXddd-d(d9dddd?dcd#ddddgyZdS(<) annotationsN)datetime)Decimal) byteorder) TYPE_CHECKINGCallableContextManagerCounterIterable)can_set_locale get_locales set_locale)Dtype)pa_version_under1p01)is_float_dtypeis_integer_dtype is_sequenceis_unsigned_integer_dtype pandas_dtype) CategoricalCategoricalIndex DataFrame DatetimeIndexIndex IntervalIndex MultiIndex RangeIndexSeries bdate_range)closenetworkround_trip_localpathround_trip_pathlibround_trip_picklewrite_to_compressed)randboolrands rands_array)assert_produces_warningmaybe_produces_warning)assert_almost_equalassert_attr_equalassert_categorical_equalassert_class_equalassert_contains_all assert_copyassert_datetime_array_equalassert_dict_equal assert_equalassert_extension_array_equalassert_frame_equalassert_index_equal!assert_indexing_slices_equivalentassert_interval_array_equalassert_is_sorted"assert_is_valid_plot_return_objectassert_metadata_equivalentassert_numpy_array_equalassert_period_array_equalassert_series_equalassert_sp_array_equalassert_timedelta_array_equalraise_assert_detail) get_dtypeget_obj) RNGContextdecompress_file ensure_cleanensure_clean_dir!ensure_safe_environment_variables set_timezone use_numexprwith_csv_dialect) Float64Index Int64Index NumericIndex UInt64Index)BaseMaskedArrayExtensionArray PandasArray)NDArrayBackedExtensionArray) extract_array) PeriodIndexTimedeltaIndexuint8uint16uint32uint64z list[Dtype]UNSIGNED_INT_NUMPY_DTYPESZUInt8ZUInt16ZUInt32ZUInt64UNSIGNED_INT_EA_DTYPESint8int16int32int64SIGNED_INT_NUMPY_DTYPESZInt8ZInt16ZInt32ZInt64SIGNED_INT_EA_DTYPESfloat32float64FLOAT_NUMPY_DTYPESZFloat32ZFloat64FLOAT_EA_DTYPESZ complex64Z complex128COMPLEX_DTYPESstrU STRING_DTYPESzdatetime64[ns]zM8[ns]DATETIME64_DTYPESztimedelta64[ns]zm8[ns]TIMEDELTA64_DTYPESbool BOOL_DTYPESbytes BYTES_DTYPESobject OBJECT_DTYPES<>)littlebignanNaNcCs g|]}dD]}|d|q qS)) YMWDhmsmsusnsZpsfsasNaT).0clsunitrrsrrrrrcCs$g|]}dD]}tj||dq qS))NUTCz US/Pacificz US/Eastern)rtz)pa timestamp)rrrrrrrs)rrrrcCsg|]}t|qSr)rduration)rrrrrrsz^$None)returncCs0tjdd}d|kr,tD]}td|qdS)NPANDAS_TESTING_MODEr deprecatealwaysosenvironget_testing_mode_warningswarnings simplefilterZ testing_modecategoryrrrset_testing_modesrcCs0tjdd}d|kr,tD]}td|qdS)Nrrrignorerrrrrreset_testing_modesrcCstjddddS)zJ Reset the display options for printing and representing objects. z ^display.T)ZsilentN)pdZ reset_optionrrrrreset_display_optionssrcCst|t|kS)zO Checks if the set of unique elements of arr1 and arr2 are equivalent. ) frozenset)Zarr1Zarr2rrr equalContentssrTcCs|tjkr2t|tr&tt|j}qt|}n|tkrFt |}n|t krXt |}nn|t krt | }|r|j }tj|gddd}n:|tjks|tjkrt|}n|tkrt|}nt||S)z Helper function to wrap the expected output of a test in a given box_class. Parameters ---------- expected : np.ndarray, Index, Series box_cls : {Index, Series, DataFrame} Returns ------- subclass of box_cls T)Z ignore_index)rarray isinstancerrQnpasarray_valuesrZ _with_inferrrZto_frameTconcatndarrayto_arrayNotImplementedError)expectedZbox_clsZ transposerrr box_expecteds&        rcCs*t|dd}|dkrt|St|ddS)zQ Similar to pd.array, but does not cast numpy dtypes to nullable dtypes. dtypeNT)Z extract_numpy)getattrrrrS)objrrrrr?s  rcCstjd|SN)stringascii_uppercasekrrrgetColsPsr rcCsttd|d|dS)Nr)ncharssizename)rr'rrrrrmakeStringIndexUsrrcKs6td|dd}ttjt|||dfd|i|S)z%make a length k index or n categoriesrWF)rrreplace) categoriesr)r'rrZ from_codesrarange)rnrkwargsxrrrmakeCategoricalIndexYsrrcKs*tjdd|dd}tj|fd|i|S)zmake a length k IntervalIndexrd)numr)rZlinspacerZ from_breaks)rrrrrrrmakeIntervalIndexasrcCsL|dkrtdg|dS|dkr.tddg|dStddgdg|d|dS)NrTrrF)rrrrr makeBoolIndexgs rrMcCst|}t|tjstt|rNtj||d}t|r|d|jdd7}nNt |rtj |tj d}| |dtj dd}ntd|t|||d S) Nrrrrr z wrong dtype )rr)rrrrAssertionErrorrrritemsizerrandomZ random_samplesortrandintrrM)rrrvaluesrrrmakeNumericIndexosrrLcCst||dd}t|S)Nrarr)rrLrrZbase_idxrrr makeIntIndexsrrNcCst||dd}t|S)Nr[r)rrNrrrr makeUIntIndexsrrcKstd|dfd|i|S)Nrrr)r)rrrrrrmakeRangeIndexsrrKcCst||dd}t|S)Nrer)rrKrrrrmakeFloatIndexsrBintr)rrcKs0tddd}t||||d}t|fd|i|S)Nr)periodsfreqrr)rrr)rrrrdtZdrrrr makeDateIndexs rr}rUcKstjfd|||d|S)Nz1 daystartrrr)rZtimedelta_range)rrrrrrrmakeTimedeltaIndexsrrTcKs&tddd}tjf||d|d|S)Nrrrr)rrZ period_range)rrrrrrrmakePeriodIndexs rcKsJ|dd}t|}tjd|gfd|i|}t||ks>t|d|S)Nrr)Zfoobarnames)rangerZ from_productlenr)rrrNrngmirrrmakeMultiIndexs  rccs tttttttg}|EdHdSr)rrrrrrr)make_index_funcsrrrindex_subclass_makers_generators rzIterable[Index]ccs$tttg}|D]}||dVqdS)z Generator which can be iterated over to get instances of all the classes which represent time-series. Parameters ---------- k: length of each of the index instances rN)rrr)rrZmake_index_funcrrrall_timeseries_index_generators rrc CsHtt}tjt}tjdd|j|dd}W5QRXt|||dS)Nr)invalidF)copyindexr)r_NrrrandnZerrstateastyper)rrrdatarrrmake_rand_seriess  rcCs t|dSNrrrrrrmakeFloatSeriessrcCs t|dSrrrrrrmakeStringSeriessrcCs*tt}t|td}tt}t|||dS)Nrr)rrrrrr)rrrrrrmakeObjectSeriess rzdict[str, Series]csttfddttDS)Ncs"i|]}|ttjtdqS)r)rrrrrrcrrr sz!getSeriesData..)rrr_Krrrr getSeriesDatasr cCs*|dkr t}ttj|t||d|dS)N)rr)rrrrrr)nperrrrrrmakeTimeSeriess r cCs&|dkr t}ttj|t||dS)Nr)rrrrrr)r rrrrmakePeriodSeriessr csfddttDS)Ncsi|]}|tqSr)r rrr rrrsz%getTimeSeriesData..rr )r rrrrgetTimeSeriesDatasrcsfddttDS)Ncsi|]}|tqSr)r rr rrrsz!getPeriodData..rrrrr getPeriodDatasrrcCst||}t|Sr)rr)r rrrrrmakeTimeDataFrames rcCst}t|Sr)r r)rrrr makeDataFramesrcCsNtdddddg}dddd d gdddddgd d d ddgtdddd}||fS)Nabrdeg?g@g@g@Zfoo1Zfoo2Zfoo3Zfoo4Zfoo5z1/1/2009)r)ArCr})rr)rrrrrgetMixedTypeDicts    rcCsttdS)Nr)rrrrrrmakeMixedDataFramesrcCst|}t|Sr)rr)r rrrrmakePeriodFramesr#Fzbool | str | list[str] | None)rrcsd|dkrdg|}t|r&t||ks*t|dksR|dksR|dksRt||ksRt|dksn|dkrj|dksnt|dkrfddt|D}|dkrd}t|tr|dkr|g}tttt t t d}| |}|r||}|r|d |_ |S|dk rtt|d t||kr,|dg|t|t||ks>ttd d |DsVtg} t|D]x} d d} ||| d} t} t| D]&}d| d|}|| | |<qt| | dd|}| |qbtt| }|dkr|dkrdn|d }t|d |d}nF|dkrR|dkr2dn|d }tdd |D|d}ntj||d}|S)a Create an index/multindex with given dimensions, levels, names, etc' nentries - number of entries in index nlevels - number of levels (> 1 produces multindex) prefix - a string prefix for labels names - (Optional), bool or list of strings. if True will use default names, if false will use no names, if a list is given, the name of each level in the index will be taken from the list. ndupe_l - (Optional), list of ints, the number of rows for which the label will repeated at the corresponding level, you can specify just the first few, the rest will use the default ndupe_l of 1. len(ndupe_l) <= nlevels. idx_type - "i"/"f"/"s"/"dt"/"p"/"td". If idx_type is not None, `idx_nlevels` must be 1. "i"/"f" creates an integer/float index, "s" creates a string "dt" create a datetime index. "td" create a datetime index. if unspecified, string labels will be generated. NrFT)ifrurptdcsg|]}t|qSr)ri)rr!prefixrrrFsz#makeCustomIndex..)r!r"rrr%r$rzE is not a legal value for `idx_type`, use 'i'/'f'/'s'/'dt'/'p'/'td'.css|]}|dkVqdSrNrrrrrr isz"makeCustomIndex..cSs*ddl}|dd|d}dd|DS)Nrz[^\d_]_?_cSsg|] }t|qSr)r)rrrrrrrsz4makeCustomIndex..keyfunc..)resubsplit)rr-Z numeric_tuplerrrkeyfuncnsz makeCustomIndex..keyfuncZ_lZ_g)keyrcss|]}|dVqdSr(rr)rrrr*s)r)rrrrrrirrrrrrrr ValueErrorreprextendall collectionsr sortedelementsappendlistziprr from_tuples)Znentriesnlevelsr'rndupe_lidx_typeZ idx_func_dictZidx_funcidxZ list_of_listsr!r0Z div_factorZcntjlabelresultZtuplesrrrr&rmakeCustomIndexsn (         rDrc s|dks t|dkst| dks4| dkr0|dks4t| dksP| dkrL|dksPtt|d||| d} t||d||| d} dkrdd fd d t|D}t|| | | d S) a Create a DataFrame using supplied parameters. Parameters ---------- nrows, ncols - number of data rows/cols c_idx_names, idx_names - False/True/list of strings, yields No names , default names or uses the provided names for the levels of the corresponding index. You can provide a single string when c_idx_nlevels ==1. c_idx_nlevels - number of levels in columns index. > 1 will yield MultiIndex r_idx_nlevels - number of levels in rows index. > 1 will yield MultiIndex data_gen_f - a function f(row,col) which return the data value at that position, the default generator used yields values of the form "RxCy" based on position. c_ndupe_l, r_ndupe_l - list of integers, determines the number of duplicates for each label at a given level of the corresponding index. The default `None` value produces a multiplicity of 1 across all levels, i.e. a unique index. Will accept a partial list of length N < idx_nlevels, for just the first N levels. If ndupe doesn't divide nrows/ncol, the last label might have lower multiplicity. dtype - passed to the DataFrame constructor as is, in case you wish to have more control in conjunction with a custom `data_gen_f` r_idx_type, c_idx_type - "i"/"f"/"s"/"dt"/"td". If idx_type is not None, `idx_nlevels` must be 1. "i"/"f" creates an integer/float index, "s" creates a string index "dt" create a datetime index. "td" create a timedelta index. if unspecified, string labels will be generated. Examples -------- # 5 row, 3 columns, default names on both, single index on both axis >> makeCustomDataframe(5,3) # make the data a random int between 1 and 100 >> mkdf(5,3,data_gen_f=lambda r,c:randint(1,100)) # 2-level multiindex on rows with each label duplicated # twice on first level, default names on both axis, single # index on both axis >> a=makeCustomDataframe(5,3,r_idx_nlevels=2,r_ndupe_l=[2]) # DatetimeIndex on row, index with unicode labels on columns # no names on either axis >> a=makeCustomDataframe(5,3,c_idx_names=False,r_idx_names=False, r_idx_type="dt",c_idx_type="u") # 4-level multindex on rows with names provided, 2-level multindex # on columns with default labels and default names. >> a=makeCustomDataframe(5,3,r_idx_nlevels=4, r_idx_names=["FEE","FIH","FOH","FUM"], c_idx_nlevels=2) >> a=mkdf(5,3,r_idx_nlevels=2,c_idx_nlevels=4) rN)r!r"rrr$r%rr)r=r'rr>r?RcSsd|d|S)NrErr)rrrrrz%makeCustomDataframe..cs$g|]fddtDqS)csg|]}|qSrrr) data_gen_frFrrrsz2makeCustomDataframe...)r)rrIncols)rFrrsz'makeCustomDataframe..r)rrDrr)nrowsrKZ c_idx_namesZ r_idx_namesZ c_idx_nlevelsZ r_idx_nlevelsrIZ c_ndupe_lZ r_ndupe_lrZ c_idx_typeZ r_idx_typecolumnsrrrrJrmakeCustomDataframes@H   rNc s|dkrtj}n tj|}td|d}d}t||}fdd}|||}|jkr|d9}|||}qdt|dt} || t} | | fS)NrrgRQ?cs.|t|}tt|dSr)Zrandrruniquefloor)rZ _extra_sizeindrKrLrrr_gen_unique_randsz-_create_missing_idx.._gen_unique_randg?r) rrZ RandomStateroundminrrPrrtolist) rLrKdensity random_stateZmin_rowsZfac extra_sizerSrQrAr!rrRr_create_missing_idxs    rZ?cCs0t}t|j||d\}}tj|j||f<|S)N)rWrX)rrZshaperrxr)rWrXZdfr!rArrrmakeMissingDataframesr]c@s,eZdZddgZeddZeddZdS)SubclassedSeriestestattrrcCsddS)Nc_s t||Srr^argsrrrrrG'rHz/SubclassedSeries._constructor..rselfrrr _constructor szSubclassedSeries._constructorcCsddS)Nc_s t||SrSubclassedDataFramerarrrrG+rHz9SubclassedSeries._constructor_expanddim..rrcrrr_constructor_expanddim)sz'SubclassedSeries._constructor_expanddimN)__name__ __module__ __qualname__ _metadatapropertyrerhrrrrr^s  r^c@s*eZdZdgZeddZeddZdS)rgr_cCsddS)Nc_s t||SrrfrarrrrG3rHz2SubclassedDataFrame._constructor..rrcrrrre1sz SubclassedDataFrame._constructorcCsddS)Nc_s t||Srr`rarrrrG7rHz9SubclassedDataFrame._constructor_sliced..rrcrrr_constructor_sliced5sz'SubclassedDataFrame._constructor_slicedN)rirjrkrlrmrernrrrrrg.s  rgc@seZdZeddZdS)SubclassedCategoricalcCstSr)rorcrrrre;sz"SubclassedCategorical._constructorN)rirjrkrmrerrrrro:srocs"rfdd}n fdd}|S)a` Create a function for calling on an array. Parameters ---------- alternative : function The function to be called on the array with no NaNs. Only used when 'skipna_alternative' is None. skipna_alternative : function The function to be called on the original array Returns ------- function cs |jSr)rr)skipna_alternativerrskipna_wrapperRsz,_make_skipna_wrapper..skipna_wrappercs"|}t|dkrtjS|S)Nr)Zdropnarrrx)rZnona) alternativerrrrWs r)rsrqrrr)rsrqr_make_skipna_wrapper@s rtz list[str]) rows_listrcCstj}|||S)aW Convert list of CSV rows to single CSV-formatted string for current OS. This method is used for creating expected value of to_csv() method. Parameters ---------- rows_list : List[str] Each element represents the row of csv. Returns ------- str Expected output of to_csv() in current OS. )rlinesepjoin)ruseprrrconvert_rows_list_to_csv_str`sryztype[Exception]r)expected_exceptionrcCsddl}|j|ddS)a$ Helper function to mark pytest.raises that have an external error message. Parameters ---------- expected_exception : Exception Expected error to raise. Returns ------- Callable Regular `pytest.raises` function with `match` equal to `None`. rN)match)pytestZraises)rzr|rrrexternal_error_raisedtsr}cs@g}|D]2\|f|fddtD7}q|S)a Combine frame, functions from com._cython_table keys and expected result. Parameters ---------- ndframe : DataFrame or Series func_names_and_expected : Sequence of two items The first item is a name of a NDFrame method ('sum', 'prod') etc. The second item is the expected return value. Returns ------- list List of three items (DataFrame, function, expected result) cs"g|]\}}|kr|fqSrr)rfuncrr func_namendframerrrsz+get_cython_table_params..)r9 cython_table)rZfunc_names_and_expectedresultsrrrget_cython_table_paramss  rr)op_namercsP|d}ztt|}Wn2tk rJtt|ddfdd}YnX|S)z The operator function for a given op name. Parameters ---------- op_name : str The op name, in form of "add" or "__add__". Returns ------- function A function performing the operation. r,rNcs ||Srr)ryZroprrrGrHz"get_op_from_name..)striproperatorAttributeError)rZ short_opnameoprrrget_op_from_names rcCs|SrrrprrrgetitemsrcCs|SrrrprrrsetitemsrcCs|jSr)locrprrrrsrcCs|jSr)ilocrprrrrsrcCs|jSr)atrprrrrsrcCs|jSr)iatrprrrrsrcCst|tjr$t|tjr$t||St|tjr:t||St|trHdSt|tr^t|j|St|ttfrxt|j |St|t rt|j |St|t j jjrt|j|St|t j jjrt|j|pt|j|St|tr:|jdkr:t|tr:|jdkr:|j}|j}|dd}|dd}||kSt|trtt|trtt|j|jprt|j|jSt|trt|jjdkr|jjd}t||Stt|t|dS)z- Pandas-compat for np.shares_memory. Fzstring[pyarrow]rrN) rrr shares_memoryrr_codesrrrrRZ_ndarrayrcoreZarraysZ SparseArrayZ sp_valuesZ IntervalArray_leftZ_rightrPr_datachunkbuffersrOZ_maskrrZ_mgrrtype)leftrightZ left_pa_dataZ right_pa_dataZ left_buf1Z right_buf1Zarrrrrrs@            rALL_INT_EA_DTYPESALL_INT_NUMPY_DTYPESALL_NUMPY_DTYPESALL_REAL_NUMPY_DTYPESr*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r(r=r>r?r rrDEMPTY_STRING_PATTERNENDIANrErFrGrAr rBr)NARROW_NP_DTYPESr NP_NAT_OBJECTS NULL_OBJECTSr@r%r&rCr!r"r#r rHrIrJr$)T)rN)rrN)rN)rN)rN)rN)rN)rN)rN)rrN)rr}N)rN)rN)r)N)N)N)NrN)NN)Nr)N)Nr)N)r FNN) TTrrNNNNNN)N)r[N)N) __future__rr6rdecimalrrrr-rsysrtypingrrrr r rZnumpyrZpandas._config.localizationr r r Zpandas._typingrZ pandas.compatrZpandas.core.dtypes.commonrrrrrZpandasrrrrrrrrrrrZpandas._testing._iorr r!r"r#r$Zpandas._testing._randomr%r&r'Zpandas._testing._warningsr(r)Zpandas._testing.assertersr*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@Zpandas._testing.compatrArBZpandas._testing.contextsrCrDrErFrGrHrIrJZpandas.core.apirKrLrMrNZpandas.core.arraysrOrPrQZpandas.core.arrays._mixinsrRZpandas.core.constructionrSrTrUrr r\__annotations__r]rrbrcrrfloatrfrgcomplexrhrirkrlrmrnrorprqrrrsrrZfloat16rdr^r_r`rXrYrZrrrxrZNArZ datetime64Z timedelta64rZpyarrowrr[ZUNSIGNED_INT_PYARROW_DTYPESraZSIGNED_INT_PYARROW_DTYPESZALL_INT_PYARROW_DTYPESreZFLOAT_PYARROW_DTYPESutf8ZSTRING_PYARROW_DTYPESZtime32Ztime64ZTIME_PYARROW_DTYPESZdate32Zdate64ZDATE_PYARROW_DTYPESZDATETIME_PYARROW_DTYPESZTIMEDELTA_PYARROW_DTYPESZbool_ZBOOL_PYARROW_DTYPESZALL_PYARROW_DTYPEScompilerDeprecationWarningResourceWarningrrrrrrrrrrrrrrrrrrrrrrrrrrrr r r rrrrrrrrDrNrZr]r^rgrortryr}rcommonZ _cython_tableitemsrrrrrrrrrr__all__rrrrsb      0 d(          )   uk   2