o
    id                     @  s  d dl mZ d dlZd dlZd dlmZmZmZ d dlZ	d dl
mZmZ d dlmZ d dlmZmZmZmZ d dlmZ d dlmZmZ d d	lmZmZmZmZ d d
lm Z m!Z! d dl"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*  m+Z, d dl-m.Z.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; erd dl<m=Z= d dl7m>Z> dHddZ?dId"d#Z@dHd$d%ZAdJd(d)ZBdKd.d/ZCG d0d1 d1ZDdLd4d5ZEdMd9d:ZFdNd<d=ZGdOd>d?ZHdPd@dAZIdQdDdEZJdRdFdGZKdS )S    )annotationsN)TYPE_CHECKINGSequencecast)NaT	internals)NA)	ArrayLikeDtypeObjManagerShape)cache_readonly)ensure_dtype_can_hold_nafind_common_type)is_1d_only_ea_dtypeis_dtype_equal	is_scalarneeds_i8_conversion)cast_to_common_typeconcat_compat)DatetimeTZDtypeExtensionDtype)is_valid_na_for_dtypeisnaisna_all)DatetimeArrayExtensionArray)SparseDtype)ensure_wrapped_if_datetimelike)ArrayManagerNullArrayProxy)ensure_block_shapenew_block_2d)BlockManager)Index)Blockaxeslist[Index]concat_axisintcopyboolreturnr   c              	     s   g  | D ]6\}}d}|  D ]\}}|j|| ||ddd}|dkr(|dur(d}q|r5|dkr5|s5| } | q|dkrP fddtt d jD }	n|dksVJ ttj	
d	d  D }	t|	|d |d gdd
}
|
S )z
    Concatenate array managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (ArrayManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    ArrayManager
    FT)axis
allow_dupsuse_na_proxy   Nr   c                   s,   g | ] t  fd dttD qS )c                   s   g | ]	}| j   qS  arrays).0i)jmgrsr1   W/var/www/edux/Edux_v2/venv/lib/python3.10/site-packages/pandas/core/internals/concat.py
<listcomp>l       z:_concatenate_array_managers.<locals>.<listcomp>.<listcomp>)concat_arraysrangelen)r4   r7   )r6   r8   r9   k   s    z/_concatenate_array_managers.<locals>.<listcomp>c                 S     g | ]}|j qS r1   r2   )r4   mgrr1   r1   r8   r9   r       )verify_integrity)itemsreindex_indexerr*   appendr<   r=   r3   list	itertoolschainfrom_iterabler   )mgrs_indexersr&   r(   r*   r@   indexersaxis1_made_copyaxindexerr3   new_mgrr1   r>   r8   _concatenate_array_managersH   s*   
rP   	to_concatrF   r	   c                   s   dd | D }dd |D }t |dk}|r|d j ntdd |D r.tt|g  n	td	d |D   fd
d| D } t| d trRt| d }|	| S t
| }t |dkrsdd |D }t |dkrsd|v rs|t}|S )a  
    Alternative for concat_compat but specialized for use in the ArrayManager.

    Differences: only deals with 1D arrays (no axis keyword), assumes
    ensure_wrapped_if_datetimelike and does not skip empty arrays to determine
    the dtype.
    In addition ensures that all NullArrayProxies get replaced with actual
    arrays.

    Parameters
    ----------
    to_concat : list of arrays

    Returns
    -------
    np.ndarray or ExtensionArray
    c                 S  s   g | ]	}t |ts|qS r1   )
isinstancer    r4   xr1   r1   r8   r9      r:   z!concat_arrays.<locals>.<listcomp>c                 S  s   h | ]}|j qS r1   dtyperS   r1   r1   r8   	<setcomp>   rA   z concat_arrays.<locals>.<setcomp>r0   r   c                 s  s&    | ]}|j d v ot|tjV  qdS ))r5   ubN)kindrR   nprV   rS   r1   r1   r8   	<genexpr>   s   $ z concat_arrays.<locals>.<genexpr>c                 S  r?   r1   rU   r4   arrr1   r1   r8   r9      rA   c                   s*   g | ]}t |tr| nt| qS r1   )rR   r    to_arrayr   r]   target_dtyper1   r8   r9      s    c                 S  s   h | ]}|j jqS r1   )rV   rZ   )r4   objr1   r1   r8   rW          rY   )r=   rV   allr[   r   rF   rR   r   type_concat_same_typeconcatenateastypeobject)rQ   to_concat_no_proxydtypessingle_dtypeclsresultkindsr1   r`   r8   r;   x   s*   



r;   c                 C  s@  t | d d trt| |||S t|| } dd | D }t||}g }|D ]s\}}|d }	|	j}
t|dkrL|d jsL|
j}|rE|	 }n|
 }d}n7t|rzdd |D }|
jsbtj|dd}nt|dd}t|dd	}t|}|
jj|jk}n	t|||d
}d}|r|
j||d}nt||d}|| q%tt||S )z
    Concatenate block managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    BlockManager
    r   c                 S  s   g | ]	\}}t ||qS r1   )_get_mgr_concatenation_plan)r4   r@   rK   r1   r1   r8   r9      s    z(concatenate_managers.<locals>.<listcomp>r0   Tc                 S  s   g | ]}|j jqS r1   )blockvaluesr4   jur1   r1   r8   r9      rc   r-      )ndim)r*   F)	placement)rR   r   rP   _maybe_reindex_columns_na_proxy_combine_concat_plansrq   r=   rK   rr   r*   view_is_uniform_join_unitsis_extensionr[   rg   r   r!   r   rV   _concatenate_join_unitsmake_block_same_classr"   rE   r#   tuple)rJ   r&   r(   r*   concat_plansconcat_planblocksrx   
join_unitsunitblkrr   fastpathvalsrY   r1   r1   r8   concatenate_managers   s@   


r   rJ   0list[tuple[BlockManager, dict[int, np.ndarray]]]c              
   C  sj   g }|D ].\}}d|v r+|j | d |d dddddd}| }|d= |||f q|||f q|S )z
    Reindex along columns so that all of the BlockManagers being concatenated
    have matching columns.

    Columns added in this reindexing have dtype=np.void, indicating they
    should be ignored when choosing a column's final dtype.
    r   FT)r-   r*   
only_slicer.   r/   )rD   r*   rE   )r&   rJ   new_mgrs_indexersr@   rK   rO   new_indexersr1   r1   r8   ry      s"   
	ry   r@   r#   rK   dict[int, np.ndarray]c                 C  sH  t | j}| D ]
\}}t|||< q	t|}d|vsJ | jr0| jd }|jt|||fgS | j	}| j
}g }	tj|ddD ]b\}
}|jsHJ |
dksNJ | }t |}t||d< t|}| j|
 }||j }t|t|jko|jjr~|jjjdkpt|dk }|r|dd n||d< t|||}|	||f q?|	S )z
    Construct concatenation plan for given block manager and indexers.

    Parameters
    ----------
    mgr : BlockManager
    indexers : dict of {axis: indexer}

    Returns
    -------
    plan : list of (BlockPlacement, JoinUnit) tuples

    r   F)groupr0   N)rF   shaperC   r=   r   is_single_blockr   mgr_locsJoinUnitblknosblklocslibinternalsget_blkno_placementsis_slice_liker*   rN   as_slicestepr[   diffrd   poprE   )r@   rK   mgr_shape_listrM   rN   	mgr_shaper   r   r   planblkno
placementsjoin_unit_indexers
shape_listr   ax0_blk_indexerunit_no_ax0_reindexingr   r1   r1   r8   rp     s<   




rp   c                   @  s`   e Zd ZddddZdd
dZedddZedddZd ddZedddZ	d!ddZ
dS )"r   Nrq   r%   r   r   c                 C  s"   |d u ri }|| _ || _|| _d S Nrq   rK   r   )selfrq   r   rK   r1   r1   r8   __init__b  s
   
zJoinUnit.__init__r,   strc                 C  s$   t | j dt| j d| j dS )N(z, ))re   __name__reprrq   rK   )r   r1   r1   r8   __repr__k  s   $zJoinUnit.__repr__r+   c                 C  s&   | j  D ]}|dk r dS qdS )Nr   TF)rK   rr   any)r   rN   r1   r1   r8   needs_fillingn  s
   zJoinUnit.needs_fillingr
   c                 C  s2   | j }|jjjdkrtd| js|jS t|jS )NVzBlock is None, no dtype)rq   rr   rV   rZ   AssertionErrorr   r   )r   r   r1   r1   r8   rV   w  s   
zJoinUnit.dtyperV   c                   s   | j sdS | jjjdkrdS | jtkr&| jj}t fdd|jddD S | jj}|t	u r6t
| j s6dS |tu r@t r@dS t| S )z
        Check that we are all-NA of a type/dtype that is compatible with this dtype.
        Augments `self.is_na` with an additional check of the type of NA values.
        Fr   Tc                 3  s    | ]}t | V  qd S r   )r   rS   rU   r1   r8   r\         z,JoinUnit._is_valid_na_for.<locals>.<genexpr>Korder)is_narq   rV   rZ   ri   rr   rd   ravel
fill_valuer   r   r   r   r   )r   rV   rr   na_valuer1   rU   r8   _is_valid_na_for  s   

zJoinUnit._is_valid_na_forc                 C  s   | j }|jjdkrdS |jsdS |j}|jdkrdS t|jtr"dS |jdkr9|d }t	|r3t
|s5dS t|S |d d }t	|rGt
|sIdS tdd |D S )Nr   TFr   r0   c                 s  s    | ]}t |V  qd S r   )r   )r4   rowr1   r1   r8   r\         z!JoinUnit.is_na.<locals>.<genexpr>)rq   rV   rZ   _can_hold_narr   sizerR   r   rw   r   r   r   rd   )r   r   rr   valr1   r1   r8   r     s&   

zJoinUnit.is_naempty_dtyper	   c                 C  s  |d u r| j jjdkr| j j}| j  }n|}| |r| j j}|tdkr;| j jjdd}t	|r;|d d u r;d }t
|trNt| j|j}t||dS t|rt||r[| jr[nXtt|}| }|jg |d}| j\}	}
|	dksxJ |	dtj|
ftjd }|j|d	|d
S t
|tr| }|j| j|d}||d d < |S tj| j|d}|| |S | js| j js| j jS | j jr| j tdj}n| j j}| js|  }|S | j! D ]\}}t"j#|||d}q|S )Nr   ri   r   r   r   rU   r0   r   T)
allow_fillr   )r   rV   ru   )$rq   rV   rZ   r   
get_valuesr   r[   rr   r   r=   rR   r   fullr   valuer   r   r   rK   r   r   construct_array_type_from_sequenceonesintptake_emptyemptyfill_can_consolidateis_boolrh   r{   rC   algostake_nd)r   r   upcasted_nar   rr   	blk_dtypei8valuesrm   missing_arrncolsnrows	empty_arrrM   rN   r1   r1   r8   get_reindexed_values  sX   





zJoinUnit.get_reindexed_valuesr   )rq   r%   r   r   )r,   r   r,   r+   )r,   r
   )rV   r
   r,   r+   )r   r
   r,   r	   )r   
__module____qualname__r   r   r   r   rV   r   r   r   r1   r1   r1   r8   r   a  s    
	
	r   r   list[JoinUnit]c                   s   |dkrt | dkrtdt|  tdd | D }t | fdd| D }t |dkrM|d }|rKt|tjrG|jdurE|	 }|S |	 }|S td	d |D rkd
d |D }t
|ddd}t|d}|S t
||d}|S )zI
    Concatenate values from several join units along selected axis.
    r   r0   z$Concatenating join units along axis0c                 s      | ]
}|j jjd kV  qdS r   Nrq   rV   rZ   r4   r   r1   r1   r8   r\         z*_concatenate_join_units.<locals>.<genexpr>c                   s   g | ]	}|j  d qS )r   r   )r   rs   r   r1   r8   r9     s    z+_concatenate_join_units.<locals>.<listcomp>Nc                 s  s    | ]}t |jV  qd S r   r   rV   r4   tr1   r1   r8   r\   /  r   c                 S  s*   g | ]}t |jr|n|d ddf qS )r   Nr   r   r1   r1   r8   r9   7  s    T)r-   ea_compat_axisrv   ru   )r=   r   _get_empty_dtyper   _dtype_to_na_valuerR   r[   ndarraybaser*   r   r!   )r   r(   r*   has_none_blocksrQ   concat_valuesr1   r   r8   r~     s4   


r~   rV   r
   r   c                 C  sr   t | tr| jS | jdv r| dS | jdv r| dS | jdkr#dS | jdv r/|s,dS tjS | jdkr7tjS t)	z2
    Find the NA value to go with this dtype.
    )mMr   )fcNaNrY   N)r5   rX   O)rR   r   r   rZ   re   r[   nanNotImplementedError)rV   r   r1   r1   r8   r   F  s   







r   Sequence[JoinUnit]c                 C  s   t | dkr| d j}|jS t| r| d jj}|S tdd | D }dd | D }t |s5dd | D }t|}|r?t|}|S )z
    Return dtype and N/A values to use when concatenating specified units.

    Returned N/A value may be None which means there was no casting involved.

    Returns
    -------
    dtype
    r0   r   c                 s  r   r   r   r   r1   r1   r8   r\   o  r   z#_get_empty_dtype.<locals>.<genexpr>c                 S  s   g | ]}|j s|jqS r1   )r   rV   r   r1   r1   r8   r9   q  s    z$_get_empty_dtype.<locals>.<listcomp>c                 S  s    g | ]}|j jjd kr|jqS )r   r   r   r1   r1   r8   r9   s  s     )r=   rq   rV   _is_uniform_reindexr   r   r   )r   r   r   r   rk   rV   r1   r1   r8   r   ]  s   

r   c                   sv   | d j   jjdkrdS t fdd| D o:t fdd| D o:tdd | D o:tdd | D o:t| d	kS )
z
    Check if the join units consist of blocks of uniform type that can
    be concatenated using Block.concat_same_type instead of the generic
    _concatenate_join_units (which uses `concat_compat`).

    r   r   Fc                 3  s"    | ]}t |jt  u V  qd S r   )re   rq   rs   firstr1   r8   r\     s     z)_is_uniform_join_units.<locals>.<genexpr>c                 3  s.    | ]}t |jj jp|jjjd v V  qdS ))rY   r5   rX   N)r   rq   rV   rZ   rs   r   r1   r8   r\     s    
c                 s  s     | ]}|j  p|jjV  qd S r   )r   rq   r}   rs   r1   r1   r8   r\     s    c                 s  s    | ]}|j  V  qd S r   )rK   rs   r1   r1   r8   r\     r   r0   )rq   rV   rZ   rd   r=   r   r1   r   r8   r|   {  s   

r|   c                 C  s(   t dd | D otdd | D dkS )Nc                 s  s    | ]}|j jV  qd S r   )rq   r}   rs   r1   r1   r8   r\     r   z&_is_uniform_reindex.<locals>.<genexpr>c                 S  s   h | ]}|j jjqS r1   )rq   rV   namers   r1   r1   r8   rW     s    z&_is_uniform_reindex.<locals>.<setcomp>r0   )rd   r=   r   r1   r1   r8   r     s   r   	join_unitlengthc                 C  s   d| j vr#| j }| jdu rd}n2| jt|d}| jt|| _n| j}t| j }|d |d |d< | j d d| | j d< | jd | f| jdd  }|f| jdd  | _t|||dS )z
    Reduce join_unit's shape along item axis to length.

    Extra items that didn't fit are returned as a separate block.
    r   Nr0   r   )rK   rq   getitem_blockslicer*   r   r   )r   r   extra_indexersextra_blockextra_shaper1   r1   r8   _trim_join_unit  s   

r  c                 #  s   t | dkr| d D ]}|d |d gfV  qdS |dkrEd}| D ] }d}|D ]\}}|||gfV  |}q(|durB||jj7 }q"dS dg  fdd}ttt| } tt|| }	 d t |	krވ d dkrntdt|	 \}
}ttt |
}t	|t
|}}||kr|
d |fV  t|| |	dd< n;d}dgt |	 }t|	D ](\}\}}|||< t ||kr||d t||f|	|< q|}|| | |	|< q||fV   d t |	ksddS dS )z`
    Combine multiple concatenation plans into one.

    existing_plan is updated in-place.
    r0   r   Nc                   s&   t | d }|d u r d  d7  < |S )Nr   r0   )next)seqretval	num_endedr1   r8   _next_or_none  s   
z,_combine_concat_plans.<locals>._next_or_nonezPlan shapes are not aligned)r=   addr   stoprF   mapiter
ValueErrorzipminmax	enumerater  )plansr(   poffsetr   last_plcplcr   r  
next_itemsr   unitslengthsmin_lenmax_lenyielded_placementyielded_unitsr5   r1   r	  r8   rz     sN   
rz   )r&   r'   r(   r)   r*   r+   r,   r   )rQ   rF   r,   r	   )r&   r'   rJ   r   r,   r   )r@   r#   rK   r   )r   r   r(   r)   r*   r+   r,   r	   )rV   r
   r   r+   )r   r   r,   r
   )r   r   r,   r+   r   )r   r   r   r)   r,   r   )r(   r)   )L
__future__r   r*   rG   typingr   r   r   numpyr[   pandas._libsr   r   r   pandas._libs.missingr   pandas._typingr	   r
   r   r   pandas.util._decoratorsr   pandas.core.dtypes.castr   r   pandas.core.dtypes.commonr   r   r   r   pandas.core.dtypes.concatr   r   pandas.core.dtypes.dtypesr   r   pandas.core.dtypes.missingr   r   r   pandas.core.algorithmscore
algorithmsr   pandas.core.arraysr   r   pandas.core.arrays.sparser   pandas.core.constructionr   #pandas.core.internals.array_managerr   r    pandas.core.internals.blocksr!   r"   pandas.core.internals.managersr#   pandasr$   r%   rP   r;   r   ry   rp   r   r~   r   r   r|   r   r  rz   r1   r1   r1   r8   <module>   sL    

0
8
F
 K 
/
7


#
