o
    i                     @   s  d Z ddlZddlmZ ddlZddlZddlmZmZ ddl	Z
ddlZddlmZ ddlmZ ddlmZmZmZmZ ddlm  mZ ddlZddlmZ ddlmZ dd	l m!Z!m"Z"m#Z#m$Z$m%Z% zddl&Z&d
Z'W n e(yv   dZ'Y nw z)e  edde)d edde*d ddl+Z+W d   n1 sw   Y  d
Z,W n e(y   dZ,Y nw ej-dZ.ej/ej0dej-j1e, peddkdddej0dej-j1e' dddgddd Z2ej/dd Z3ej/dd Z4ej/d d! Z5ej/d"d# Z6ej/d$d% Z7ej/ej8ej9j:ej8ej9j;ej8ej9j<ej=d&d'ej=d(d'ej=d)d'ej=d*d'gdd+d, Z>						
		
	-dKd.d/Z?d0d1 Z@d2d3 ZAd4d5 ZBd6d7 ZCd8d9 ZDd:d; ZEd<d= ZFd>d? ZGd@dA ZHG dBdC dCZIG dDdE dEeIZJej-dFG dGdH dHeIZKG dIdJ dJeIZLdS )Lz test parquet compat     N)BytesIO)catch_warningsfilterwarnings)
get_option)is_platform_windows)pa_version_under2p0pa_version_under5p0pa_version_under6p0pa_version_under8p0)Version)FastParquetImplPyArrowImpl
get_engineread_parquet
to_parquetTFignorez	`np.bool`)categoryz.*Int64Index.*z4ignore:RangeIndex.* is deprecated:DeprecationWarningfastparquetmode.data_managerarrayz4fastparquet is not installed or ArrayManager is usedreason)markspyarrowpyarrow is not installed)paramsc                 C      | j S Nparamrequest r"   W/var/www/edux/Edux_v2/venv/lib/python3.10/site-packages/pandas/tests/io/test_parquet.pyengineB   s   r$   c                   C   s   t std dS )Nr   r   )_HAVE_PYARROWpytestskipr"   r"   r"   r#   paW   s   
r(   c                   C   s,   t s	td dS tddkrtd dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )_HAVE_FASTPARQUETr&   r'   r   r"   r"   r"   r#   fp^   s   

r*   c                   C   s   t g dddS )N         fooAB)pd	DataFramer"   r"   r"   r#   	df_compatg   s   r5   c               
   C   sB   t tdttddtjddddg dt jd	d
dd} | S )Nabcr,            @      @float64dtypeTFT20130101r.   periods)abdef)r3   r4   listrangenparange
date_range)dfr"   r"   r#   df_cross_compatl   s   rL   c                   C   s   t tddtjdgg dg dg dttddtd	d
dtjdddddtjdgg dt jdd	dt 	dt j
t 	dgdS )Nr6   rA   crA   NrM   )   foo   bars   bazr/   barbazr,   r7   r.      u1r8   r9   r:   r;          @      @r=   r>   r?   20130103)stringstring_with_nanstring_with_nonebytesunicodeintuintfloatfloat_with_nanbooldatetimedatetime_with_nat)r3   r4   rF   rH   nanrG   rI   astyperJ   	TimestampNaTr"   r"   r"   r#   df_full~   s$   

ri   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   r   r   r   r    r"   r"   r#   timezone_aware_date_list   s   rj   r-   c
                    s   pddip	i du r|r|d< |d<  fdd}
du rFt  |
|	 W d   dS 1 s?w   Y  dS |
|	 dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    compressionNr$   c              	      sr   t | D ]2}jfi  tdd tfi }W d    n1 s'w   Y  tj| d qd S )NTrecord)check_names
check_likecheck_dtype)rG   r   r   r   tmassert_frame_equal)repeat_actualrp   ro   rn   rK   expectedpathread_kwargswrite_kwargsr"   r#   compare   s   z!check_round_trip.<locals>.compare)rq   ensure_clean)rK   r$   rx   rz   ry   rw   rn   ro   rp   rs   r{   r"   rv   r#   check_round_trip   s   "

"r}   c                 C   s~   t r'ddlm} |j| dd}t|jjt|ksJ |jjt|ks%J dS ddlm	} |j	| dd}|j
jj|ks=J dS )zCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)validate_schemahive)partitioning)r   pyarrow.parquetparquetParquetDatasetlen
partitionspartition_namessetpyarrow.datasetdatasetr   schemanames)rx   rw   pqr   dsr"   r"   r#   check_partition_names   s   
r   c                 C   sD   d}t jt|d t| dd W d    d S 1 sw   Y  d S )Nz.engine must be one of 'pyarrow', 'fastparquet'matchr/   rR   )r&   raises
ValueErrorr}   )r5   msgr"   r"   r#   test_invalid_engine   s   "r   c                 C   :   t dd t|  W d    d S 1 sw   Y  d S )Nio.parquet.enginer   r3   option_contextr}   )r5   r(   r"   r"   r#   test_options_py     
"r   c                 C   r   )Nr   r   r   )r5   r*   r"   r"   r#   test_options_fp
  r   r   c                 C   r   )Nr   autor   )r5   r*   r(   r"   r"   r#   test_options_auto  r   r   c                 C   sP  t tdts	J t tdtsJ tdd# t tdts"J t tdts+J t tdts4J W d    n1 s>w   Y  tdd# t tdtsSJ t tdts\J t tdtseJ W d    n1 sow   Y  tdd$ t tdtsJ t tdtsJ t tdtsJ W d    d S 1 sw   Y  d S )Nr   r   r   r   )
isinstancer   r   r   r3   r   )r*   r(   r"   r"   r#   test_options_get_engine  s"   "r   c                  C   s  ddl m}  | d}| d}tsdnttjt|k }ts!dnttjt|k }to.| }to3| }|s|s|r\d| d}t	j
t|d td	 W d    n1 sVw   Y  nd
}t	j
t|d td	 W d    n1 stw   Y  |rd| d}t	j
t|d td	 W d    d S 1 sw   Y  d S d}t	j
t|d td	 W d    d S 1 sw   Y  d S d S d S )Nr   )VERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r   r   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)pandas.compat._optionalr   getr%   r   r   __version__r)   r   r&   r   ImportErrorr   )r   
pa_min_ver
fp_min_verhave_pa_bad_versionhave_fp_bad_versionhave_usable_pahave_usable_fpr   r"   r"   r#   "test_get_engine_auto_error_message,  sD   






"
"r   c                 C   s   | }t  0}|j||d d t||d}t || t||ddgd}t ||ddg  W d    d S 1 s9w   Y  d S )Nr$   rk   r$   rA   rC   r$   columns)rq   r|   r   r   rr   rL   r(   r*   rK   rx   resultr"   r"   r#   test_cross_engine_pa_fpW  s   
"r   c              	   C   s   | }t  M}|j||d d tdd' t||d}t || t||ddgd}t ||ddg  W d    n1 s>w   Y  W d    d S W d    d S 1 sVw   Y  d S )Nr   Trl   r   rA   rC   r   )rq   r|   r   r   r   rr   r   r"   r"   r#   test_cross_engine_fp_pae  s   
"r   c                   @   s:   e Zd Zdd Zdd Zejjejddddd	 Z	d
S )Basec              	   C   s|   t  0}tj||d t|||d d W d    n1 sw   Y  W d    d S W d    d S 1 s7w   Y  d S )Nr   rk   )rq   r|   r&   r   r   )selfrK   r$   excerr_msgrx   r"   r"   r#   check_error_on_writet  s   
"zBase.check_error_on_writec              	   C   sx   t  .}t | t|||d d W d    n1 sw   Y  W d    d S W d    d S 1 s5w   Y  d S )Nr   )rq   r|   external_error_raisedr   )r   rK   r$   r   rx   r"   r"   r#   check_external_error_on_writez  s   
"z"Base.check_external_error_on_writedhttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/parquet/simple.parquetT)urlcheck_before_testc                 C   s.   |dkr	t | d}t|}t|| d S )Nr   r   )r&   importorskipr   rq   rr   )r   r5   r$   r   rK   r"   r"   r#   test_parquet_read_from_url  s   	
zBase.test_parquet_read_from_urlN)
__name__
__module____qualname__r   r   r&   marknetworkrq   r   r"   r"   r"   r#   r   s  s    r   c                   @   s   e Zd Zdd Zdd Zdd Zejdg dd	d
 Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zejd!g d"d#d$ Zd%S )&	TestBasicc                 C   sF   t g dddt dtg dfD ]}d}| ||t| qd S )Nr+   r,   r/   r>   z+to_parquet only supports IO with DataFrames)r3   Seriesrg   rH   r   r   r   )r   r$   objr   r"   r"   r#   
test_error  s   zTestBasic.test_errorc                 C   s6   t tdttddd}ddg|_t|| d S )Nr6   r,   r7   rY   r^   r/   rR   )r3   r4   rF   rG   r   r}   )r   r$   rK   r"   r"   r#   test_columns_dtypes  s   
zTestBasic.test_columns_dtypesc                 C   s   t tdttddd}d}ddg|_| ||t| ddg|_| ||t| td	ddddtd	ddddg|_| ||t| d S )
Nr6   r,   r7   r   %parquet must have string column namesr   rO   rP   i  )r3   r4   rF   rG   r   r   r   rc   )r   r$   rK   r   r"   r"   r#   test_columns_dtypes_invalid  s   

z%TestBasic.test_columns_dtypes_invalidrk   )Ngzipsnappybrotlic                 C   sN   |dkr
t d n	|dkrt d tdg di}t||d|id d S )Nr   r   r1   r+   rk   rz   )r&   r   r3   r4   r}   )r   r$   rk   rK   r"   r"   r#   test_compression  s   
zTestBasic.test_compressionc                 C   sJ   t tdttddd}t dtdi}t|||ddgid d S )Nr6   r,   r7   r   rY   r   rw   ry   )r3   r4   rF   rG   r}   )r   r$   rK   rw   r"   r"   r#   test_read_columns  s
   
zTestBasic.test_read_columnsc                 C   s   |dk}t dg di}t|| g dt jdddtdg d	g}|D ]}||_t|t jr7|jd |_t|||d
 q%g d|_d|j_	t|| d S )Nr   r1   r+   )r-   r.   r7   r>   r.   r?   r6   )r,   r.   r7   )rn   )r   r,   r-   r/   )
r3   r4   r}   rJ   rF   indexr   DatetimeIndex
_with_freqname)r   r$   rn   rK   indexesr   r"   r"   r#   test_write_index  s    

zTestBasic.test_write_indexc                 C   s:   |}t dg di}t jg d}||_t|| d S )Nr1   r+   )rA   r,   )rA   r-   )rB   r,   )r3   r4   
MultiIndexfrom_tuplesr   r}   )r   r(   r$   rK   r   r"   r"   r#   test_write_multiindex  s
   zTestBasic.test_write_multiindexc                 C   s   |}t jdddd}t jtjdt| dtdd}t jj	d	d
g|gddgd}|j
d d}||fD ]}||_t|| t||dddgi|ddg d q4d S )Nz01-Jan-2018z01-Dec-2018MS)freqr-   r.   ABCr   Level1Level2leveldate)r   r   r1   r2   ry   rw   )r3   rJ   r4   rH   randomrandnr   rF   r   from_productcopyr   r}   )r   r(   r$   datesrK   index1index2r   r"   r"   r#   test_multiindex_with_columns  s   $
z&TestBasic.test_multiindex_with_columnsc                 C   s   t g dg dd}d dd}|jdd}t||||d t jg dg ddg d	d
}t||||d g dg dg}t jttddd tdD d|d
}|jdd}t||||d d S )Nr+   )qrs)rA   rB   F)rk   r   T)droprz   rw   )zyxwvutsr)r   rR   rR   rS   rS   r/   r/   quxr   onetwor   r   r   r   r   r      c                 S   s   g | ]}| qS r"   r"   ).0ir"   r"   r#   
<listcomp>  s    z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)r   r   )r3   r4   reset_indexr}   rF   rG   )r   r$   rK   rz   rw   arraysr"   r"   r#   test_write_ignoring_index  s    
 z#TestBasic.test_write_ignoring_indexc                 C   s@   t jg d}t jtjdd|d}d}| ||t| d S )Nr   r7   r.   r   Y\s*parquet must have string column names for all values in\s*each level of the MultiIndex)	r3   r   r   r4   rH   r   r   r   r   )r   r$   
mi_columnsrK   r   r"   r"   r#   test_write_column_multiindex  s
   z&TestBasic.test_write_column_multiindexc                 C   sP   |}g dg dg}t jtjdd|d}ddg|j_d}| ||t| d S )Nr   )r,   r-   r,   r-   r,   r-   r,   r-   r   r   r   r   r  )	r3   r4   rH   r   r   r   r   r   r   r   r(   r$   r  rK   r   r"   r"   r#   &test_write_column_multiindex_nonstring)  s   z0TestBasic.test_write_column_multiindex_nonstringc                 C   sF   |}g dg dg}t jtjdd|d}ddg|j_t|| d S )Nr   r   r   r   	ColLevel1	ColLevel2)r3   r4   rH   r   r   r   r   r}   r   r(   r$   r  rK   r"   r"   r#   #test_write_column_multiindex_string;  s   z-TestBasic.test_write_column_multiindex_stringc                 C   s:   |}g d}t jtjdd|d}d|j_t|| d S )N)rR   rS   r/   r   r   r7   r   	StringCol)r3   r4   rH   r   r   r   r   r}   r  r"   r"   r#   test_write_column_index_stringJ  s
   z(TestBasic.test_write_column_index_stringc                 C   sD   |}g d}t jtjdd|d}d|j_d}| ||t| d S )Nr,   r-   r.   r7   r   r7   r   NonStringColr   )	r3   r4   rH   r   r   r   r   r   r   r  r"   r"   r#   !test_write_column_index_nonstringV  s   z+TestBasic.test_write_column_index_nonstringc           
      C   s  dd l m} |dkrtjjdd}|j| tt	g ddt	g ddt	g dt	g d	t	g d
dt	g ddt	g ddd}t
 }||| t||d}t||dd}W d    n1 snw   Y  |d jtdksJ ttj	g dddtj	g dddtj	g dddtj	g d	ddtj	g d
ddtj	g dddtj	g dddd}	|dkr|jddd}|	jddd}	t
||	 d S )Nr   r   z.Fastparquet nullable dtype support is disabledr   r,   r-   r.   Nint64uint8)rA   rB   rM   N)TFTNr  )      ?rV   rW   Nfloat32r:   )rA   rB   rM   rC   rD   rE   gr   Tr$   use_nullable_dtypesrA   Int64r;   UInt8rY   booleanFloat32Float64rM   r,   )axis)r   r   r&   r   xfailnode
add_markerr   tabler   rq   r|   write_tabler   r<   rH   r3   r4   r   rr   )
r   r$   r!   r   r   r#  rx   result1result2rw   r"   r"   r#   test_use_nullable_dtypesb  sH   
z"TestBasic.test_use_nullable_dtypesr<   )	r  r  r  objectzdatetime64[ns, UTC]r`   z	period[D]r  rY   c                 C   sT   t dt jg |di}d }|dkrt dt jg ddi}t||ddi|d d S )Nvaluer;   r`   r  r  Tr   )r3   r4   r   r}   )r   r(   r<   rK   rw   r"   r"   r#   test_read_empty_array  s   
zTestBasic.test_read_empty_arrayN)r   r   r   r   r   r   r&   r   parametrizer   r   r   r   r   r  r  r  r  r  r  r'  r*  r"   r"   r"   r#   r     s,    
		 
1r   z8ignore:CategoricalBlock is deprecated:DeprecationWarningc                   @   s  e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	e
jje dde
jdeejgdd Zdd Ze
jjdd Ze
jjdd Ze
jjede
jddgg gdd Zedd d! Zedd"d# Zd$d% Zd&d' Ze
jdeejgd(d) Zd*d+ Zd,d- Z edd.d/ Z!ejdd0d1d2d3 Z"ejdd4d1d5d6 Z#d7d8 Z$d9d: Z%ejdd0d1d;d< Z&d=d> Z'd?S )@TestParquetPyArrowc                 C   s@   |}t jdddd}|d }||d< g d|d< t|| d S )Nr>   r.   Europe/Brusselsr@   tzdatetime_tz)TNTbool_with_none)r3   rJ   r   r}   )r   r(   ri   rK   dtir"   r"   r#   
test_basic  s   
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd|d< t|||ddg dddgid	 d S )
Nr>   r.   r-  r.  r0  rY   r^   r   r   )r3   rJ   r}   )r   r(   ri   rK   r"   r"   r#   test_basic_subset_columns  s   


z,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d}t|tsJ t|}t|}t|| d S )Nr   )r   r   r\   r   r   rq   rr   )r   r(   ri   	buf_bytes
buf_streamresr"   r"   r#   *test_to_bytes_without_path_or_buf_provided  s
   z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jtdddtdd }| ||td d S )N   r7   r.   aaar   zDuplicate column names found	r3   r4   rH   rI   reshaperF   r   r   r   r   r(   rK   r"   r"   r#   test_duplicate_columns  s   $z)TestParquetPyArrow.test_duplicate_columnsc                 C   s<   t dt jdddi}tr| ||t d S t|| d S )NrA   1 dayr.   r?   )r3   r4   timedelta_ranger
   r   NotImplementedErrorr}   r=  r"   r"   r#   test_timedelta  s   z!TestParquetPyArrow.test_timedeltac                 C   s&   t dg di}| ||tj d S )NrA   rA   r,   rV   )r3   r4   r   r   ArrowExceptionr=  r"   r"   r#   test_unsupported  s   z#TestParquetPyArrow.test_unsupportedc                 C   s6   t jddt jd}tj|dgd}| ||tj d S )Nr-   
   r;   fp16datar   )rH   rI   float16r3   r4   r   r   rD  )r   r(   rI  rK   r"   r"   r#   test_unsupported_float16  s   z+TestParquetPyArrow.test_unsupported_float16zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   	path_typec              	   C   s   t jddt jd}tj|dgd}t 2}||}ttj	 |j
||d W d    n1 s2w   Y  tj|r?J W d    d S 1 sJw   Y  d S )Nr-   rF  r;   rG  rH  )rx   r$   )rH   rI   rJ  r3   r4   rq   r|   r   r   rD  r   osrx   isfile)r   r(   rL  rI  rK   path_strrx   r"   r"   r#    test_unsupported_float16_cleanup  s   
"z3TestParquetPyArrow.test_unsupported_float16_cleanupc                 C   sd   t  }t td|d< t jg dt g dd|d< t jg dg dd	d
|d< t|| d S )NabcdefrA   )rR   r/   r/   rR   NrR   rQ   r;   rB   )rA   rB   rM   rA   rM   rB   )rB   rM   rC   T)
categoriesorderedrM   )r3   r4   CategoricalrF   CategoricalDtyper}   r=  r"   r"   r#   test_categorical  s   

z#TestParquetPyArrow.test_categoricalc                 C   s8   t d}|jdi |}d|i}t||d||d d S )Ns3fs
filesystemzpandas-test/pyarrow.parquetrx   ry   rz   r"   )r&   r   S3FileSystemr}   )r   r5   s3_resourcer(   s3sorW  s3kwr"   r"   r#   test_s3_roundtrip_explicit_fs  s   

z0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d d S )Nstorage_optionsz s3://pandas-test/pyarrow.parquetrY  r}   )r   r5   r[  r(   r\  r"   r"   r#   test_s3_roundtrip+  s   
z$TestParquetPyArrow.test_s3_roundtriprW  partition_colr1   c              
   C   sR   |  }|rtr
dnd}|| |||< t|||dd|i|d |dddd d S )	Nint32r   zs3://pandas-test/parquet_dirr`  )partition_colsrk   r`  Tr,   )rw   rx   ry   rz   ro   rs   )r   r   rf   r}   )r   r5   r[  r(   rc  r\  expected_dfpartition_col_typer"   r"   r#   test_s3_roundtrip_for_dir7  s&   
z,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t  }|| t|}t|| d S r   )r   r   r   rq   rr   )r   r5   bufferdf_from_bufr"   r"   r#   test_read_file_like_obj_supportb  s   
z2TestParquetPyArrow.test_read_file_like_obj_supportc                 C   s   | dd | dd tjtdd td W d    n1 s"w   Y  tjtdd |d W d    d S 1 s?w   Y  d S )NHOMETestingUserUSERPROFILEz.*TestingUser.*r   z~/file.parquet)setenvr&   r   OSErrorr   r   )r   r5   monkeypatchr"   r"   r#   test_expand_useri  s   
"z#TestParquetPyArrow.test_expand_userc                 C   sh   ddg}|}t   }|j||d d t|| t|j|jks"J W d    d S 1 s-w   Y  d S )Nrb   r^   re  rk   rq   ensure_clean_dirr   r   r   shape)r   r(   ri   re  rK   rx   r"   r"   r#   test_partition_cols_supportedr  s   

"z0TestParquetPyArrow.test_partition_cols_supportedc                 C   sj   d}|g}|}t   }|j||d d t|| t|j|jks#J W d    d S 1 s.w   Y  d S )Nrb   rs  rt  )r   r(   ri   re  partition_cols_listrK   rx   r"   r"   r#   test_partition_cols_string{  s   

"z-TestParquetPyArrow.test_partition_cols_stringc           	      C   sf   d}|g}|}t  }||}|j||d t|j|jks!J W d    d S 1 s,w   Y  d S )Nr2   )re  )rq   ru  r   r   rv  )	r   r(   r5   rL  re  rx  rK   rO  rx   r"   r"   r#   test_partition_cols_pathlib  s   
"z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t  }t|| d S r   )r3   r4   r}   r=  r"   r"   r#   test_empty_dataframe  s   z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }tdddgi}||jd| dg}|t}t||d|i|d d S )Nr   xr,   )typer   r   )	r   r3   r4   r   fieldbool_rf   rb   r}   )r   r(   r   rK   r   out_dfr"   r"   r#   test_write_with_schema  s
   
z)TestParquetPyArrow.test_write_with_schemac                 C   sp   t t jg dddt jg dddt jg dddd}t|| t dt jg d	ddi}t|| d S )
Nr+   r  r;   UInt32rN   rY   )rA   rB   rM   rA   r  )r3   r4   r   r}   r=  r"   r"   r#    test_additional_extension_arrays  s   
z3TestParquetPyArrow.test_additional_extension_arraysz1.0.0)min_versionc              	   C   sj   t dt jg dddi}t d| t|||d| dd W d    d S 1 s.w   Y  d S )	NrA   rN   zstring[pyarrow]r;   string_storagezstring[]rw   )r3   r4   r   r   r}   rf   )r   r(   r  rK   r"   r"   r#    test_pyarrow_backed_string_array  s   "z3TestParquetPyArrow.test_pyarrow_backed_string_arrayz2.0.0c                 C   sL   t t jg dt jddddt jt jddddd}t|| d S )N))r   r,   )r,   r-   )r.   r7   z
2012-01-01r.   D)r@   r   r7   )rM   rC   rD   )r3   r4   IntervalIndexr   period_rangefrom_breaksrJ   r}   r=  r"   r"   r#   test_additional_extension_types  s   
z2TestParquetPyArrow.test_additional_extension_typesc                 C   s>   t sd}nd}tdtjddddi}t||d|id	 d S )
Nz2.6z2.0rA   z
2017-01-011nrF  r   r@   versionr   )r	   r3   r4   rJ   r}   )r   r(   verrK   r"   r"   r#   test_timestamp_nanoseconds  s
   z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   sV   t s|jtjjkr|jtjj	dd d|g }t
j|d|id}t||dd d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r      index_as_colr   rI  F)rp   )r   tzinforc   timezoneutcr!  r"  r&   r   r   r3   r4   r}   )r   r!   r(   rj   idxrK   r"   r"   r#   test_timezone_aware_index  s   
z,TestParquetPyArrow.test_timezone_aware_indexc                 C   sr   t dttddi}t }||| t||dgdd}W d    n1 s*w   Y  t|dks7J d S )NrA   r   r.   rA   z==r   F)filtersuse_legacy_datasetr,   )	r3   r4   rF   rG   rq   r|   r   r   r   )r   r(   rK   rx   r   r"   r"   r#   test_filter_row_groups  s   

z)TestParquetPyArrow.test_filter_row_groupsc                 C   s   t jtjddg dd}t }||| t||}W d    n1 s(w   Y  |r<t	|j
t jjjs:J d S t	|j
t jjjsGJ d S )NrF  r.   )r1   r2   Cr   )r3   r4   rH   r   r   rq   r|   r   r   r   _mgrcore	internalsArrayManagerBlockManager)r   r(   using_array_managerrK   rx   r   r"   r"   r#   test_read_parquet_manager  s   
z,TestParquetPyArrow.test_read_parquet_managerN)(r   r   r   r3  r4  r8  r>  rB  rE  rK  r&   r   r   r   r+  strpathlibPathrP  rV  
single_cpur_  rb  td
skip_if_norh  rk  rr  rw  ry  rz  r{  r  r  r  r  r  r  r  r  r"   r"   r"   r#   r,    sb    

"

	






r,  c                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Ze	j
jdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!S )"TestParquetFastParquetc                 C   sF   |}t jdddd}|d }||d< t jddd|d< t|| d S )	Nr>   r.   z
US/Easternr.  r0  r?  r?   	timedelta)r3   rJ   r   r@  r}   )r   r*   ri   rK   r2  r"   r"   r#   r3    s   
z!TestParquetFastParquet.test_basicc                 C   s<   t jtdddtdd }d}| ||t| d S )Nr9  r7   r.   r:  r   z9Cannot create parquet dataset with duplicate column namesr;  r   r*   rK   r   r"   r"   r#   r>    s   $z-TestParquetFastParquet.test_duplicate_columnsc                 C   s@   t dg di}t jddtjdgidd}t|||dd d S )	NrA   )TNFr  g        rJ  r;   F)rw   rp   )r3   r4   rH   re   r}   r   r*   rK   rw   r"   r"   r#   test_bool_with_none  s   z*TestParquetFastParquet.test_bool_with_nonec                 C   sT   t dt jddddi}| ||td  t dg di}d}| ||t| d S )NrA   2013Mr.   r  rC  z"Can't infer object conversion type)r3   r4   r  r   r   r  r"   r"   r#   rE    s
   z'TestParquetFastParquet.test_unsupportedc                 C   s&   t dt tdi}t|| d S )NrA   r6   )r3   r4   rT  rF   r}   )r   r*   rK   r"   r"   r#   rV  &  s   z'TestParquetFastParquet.test_categoricalc                 C   sz   dt tddi}t|}t }|j||d dd t||dgd}W d    n1 s.w   Y  t|dks;J d S )NrA   r   r.   r,   )rk   row_group_offsetsr  )r  )	rF   rG   r3   r4   rq   r|   r   r   r   )r   r*   rC   rK   rx   r   r"   r"   r#   r  *  s   

z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dd d S )Nz$s3://pandas-test/fastparquet.parquetr`  )rk   r`  rY  ra  )r   r5   r[  r*   r\  r"   r"   r#   rb  2  s   
z(TestParquetFastParquet.test_s3_roundtripc                 C   s   ddg}|}t  -}|j|d|d d tj|sJ dd l}||dj}t	|dks/J W d    d S 1 s:w   Y  d S )Nrb   r^   r   r$   re  rk   r   Fr-   
rq   ru  r   rM  rx   existsr   ParquetFilecatsr   r   r*   ri   re  rK   rx   r   actual_partition_colsr"   r"   r#   rw  =     
"z4TestParquetFastParquet.test_partition_cols_supportedc                 C   s~   d}|}t  -}|j|d|d d tj|sJ dd l}||dj}t	|dks-J W d    d S 1 s8w   Y  d S )Nrb   r   r  r   Fr,   r  r  r"   r"   r#   ry  N  s   
"z1TestParquetFastParquet.test_partition_cols_stringc                 C   s   ddg}|}t  -}|j|dd |d tj|sJ dd l}||dj}t	|dks/J W d    d S 1 s:w   Y  d S )Nrb   r^   r   )r$   rk   partition_onr   Fr-   r  r  r"   r"   r#   test_partition_on_supported_  r  z2TestParquetFastParquet.test_partition_on_supportedc              	   C   s   ddg}|}d}t jt|d/ t }|j|dd ||d W d    n1 s)w   Y  W d    d S W d    d S 1 sAw   Y  d S )Nrb   r^   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar   r   )r$   rk   r  re  )r&   r   r   rq   ru  r   )r   r*   ri   re  rK   r   rx   r"   r"   r#   3test_error_on_using_partition_cols_and_partition_onp  s"   
"zJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s*   t  }| }d|j_t|||d d S )Nr   r  r3   r4   r   r   r   r}   r  r"   r"   r#   r{    s   z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id}| }d|j_t|||d d S )Nr  r  r  r   r  r  )r   r*   rj   r  rK   rw   r"   r"   r#   r    s
   
z0TestParquetFastParquet.test_timezone_aware_indexc              	   C   s   t dddgi}t 4}|| tjtdd t|ddd W d    n1 s,w   Y  W d    d S W d    d S 1 sDw   Y  d S )	NrA   r,   r-   z!not supported for the fastparquetr   r   Tr  )	r3   r4   rq   r|   r   r&   r   r   r   )r   r*   rK   rx   r"   r"   r#   &test_use_nullable_dtypes_not_supported  s   

"z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedc              	   C   s   t d7}t|d tjtdd t|dd W d    n1 s&w   Y  t|j	dd W d    d S 1 s?w   Y  d S )	Ntest.parquets   breakit r   r   r   F)
missing_ok)
rq   r|   r  r  write_bytesr&   r   	Exceptionr   unlink)r   rx   r"   r"   r#   $test_close_file_handle_on_read_error  s   "z;TestParquetFastParquet.test_close_file_handle_on_read_errorc              	   C   s   t jddgddgdd}td*}t| d}|| W d    n1 s*w   Y  t||d}W d    n1 s?w   Y  t|| d S )Nr   r,   r0   )rI  r  wbr   )	r3   r4   rq   r|   openencoder   r   rr   )r   r$   rK   rx   rE   r   r"   r"   r#   test_bytes_file_name  s   z+TestParquetFastParquet.test_bytes_file_nameN)r   r   r   r3  r>  r  rE  rV  r  r&   r   r  rb  rw  ry  r  r  r{  r  r  r  r  r"   r"   r"   r#   r    s$    	

	r  )	NNNNNTFTr-   )M__doc__rc   ior   rM  r  warningsr   r   numpyrH   r&   pandas._configr   pandas.compatr   pandas.compat.pyarrowr   r   r	   r
   pandas.util._test_decoratorsutil_test_decoratorsr  pandasr3   pandas._testing_testingrq   pandas.util.versionr   pandas.io.parquetr   r   r   r   r   r   r%   r   DeprecationWarningFutureWarningr   r)   r   
pytestmarkfixturer   skipifr$   r(   r*   r5   rL   ri   nowr  r  minmaxstrptimerj   r}   r   r   r   r   r   r   r   r   r   r   r   r,  r  r"   r"   r"   r#   <module>   s    








A+   
$  N