o
    i                      @   s   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ ddlmZ eeZG dd	 d	eZG d
d dZdefddZdee fddZG dd de	ZG dd dZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]

path = /path/to/huggingface_hub/.env/bin/python

args = -m debugpy --listen 5678 --wait-for-client /path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
lfs-multipart-upload ```
    N)ArgumentParser)AbstractContextManager)DictListOptional)BaseHuggingfaceCLICommand)LFS_MULTIPART_UPLOAD_COMMANDc                   @   s"   e Zd ZdZedefddZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ huggingface-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ huggingface-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    parserc                 C   sP   | j ddd}|jdtdd |jdd d	 | j td
d}|jdd d	 d S )Nzlfs-enable-largefilesz:Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                 S      t | S N)LfsEnableCommandargs r   W/var/www/edux/Edux_v2/venv/lib/python3.10/site-packages/huggingface_hub/commands/lfs.py<lambda>?       z1LfsCommands.register_subcommand.<locals>.<lambda>)funcz<Command will get called by git-lfs, do not call it directly.c                 S   r   r   )LfsUploadCommandr   r   r   r   r   E   r   )
add_parseradd_argumentstrset_defaultsr   )r
   enable_parserupload_parserr   r   r   register_subcommand6   s   zLfsCommands.register_subcommandN)__name__
__module____qualname____doc__staticmethodr   r   r   r   r   r   r	   $   s    r	   c                   @      e Zd Zdd Zdd ZdS )r   c                 C   
   || _ d S r   r   selfr   r   r   r   __init__I      
zLfsEnableCommand.__init__c                 C   sf   t j| jj}t j|std td tjd	 d|d tjdt
 	 d|d td d S )Nz)This does not look like a valid git repo.   z<git config lfs.customtransfer.multipart.path huggingface-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)osr   abspathr   isdirprintexit
subprocessrunsplitr   )r(   
local_pathr   r   r   r4   L   s   zLfsEnableCommand.runNr    r!   r"   r)   r4   r   r   r   r   r   H       r   msgc                 C   s(   t | d } tj|  tj  dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushr9   r   r   r   	write_msg^   s   rB   returnc                  C   sV   t tj  } d| d| dfv rdS | ddvr)td t	d | S )z%Read Line delimited JSON from stdin. 	terminater   eventN)downloaduploadzReceived unexpected messager+   )
r;   loadsr=   stdinreadlinestripgetloggercriticalr2   rA   r   r   r   read_msge   s   

rO   c                   @   sP   e Zd ZdZdededefddZdd Zd	d
 ZdddZ	dd Z
dd ZdS )	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    filepath	seek_from
read_limitc                 C   s   || _ || _|| _d| _d S )Nr   )rQ   rR   rS   n_seen)r(   rQ   rR   rS   r   r   r   r)   {   s   
zFileSlice.__init__c                 C   s    t | jd| _| j| j | S )Nrb)openrQ   fseekrR   r(   r   r   r   	__enter__   s   zFileSlice.__enter__c                 C   s$   t | j j}t| j|| j S r   )r.   fstatrW   filenost_sizeminrS   rR   )r(   total_lengthr   r   r   __len__   s   zFileSlice.__len__c                 C   sP   | j | jkrdS | j| j  }| j|dk r|nt||}|  j t|7  _ |S )N    r   )rT   rS   rW   readr^   len)r(   nremaining_amountdatar   r   r   rc      s   zFileSlice.readc                 c   s    | j ddV  d S )Ni  @ )re   )rc   rY   r   r   r   __iter__   s   zFileSlice.__iter__c                 G   s   | j   d S r   )rW   closer'   r   r   r   __exit__   s   zFileSlice.__exit__N)ra   )r    r!   r"   r#   r   intr)   rZ   r`   rc   rh   rj   r   r   r   r   rP   t   s    
rP   c                   @   r%   )r   c                 C   r&   r   r   r'   r   r   r   r)      r*   zLfsUploadCommand.__init__c              	   C   s  t tj  }|ddkr|ddks&tddddi td	 ti  	 t	 }|d u r7td |d }|d }|d d }|d d }t
|d}t| }td|d	dd g }	t|D ]C\}
}t||
| |d.}tj||d}|  |	|jd|
d	 d td||
d	 | |d W d    n1 sw   Y  qgtj|||	dd}|  td|d q+)NrE   init	operationrG   error    zWrong lfs init operation)codemessager+   Tr   oidr   actionhrefheader
chunk_sizeprogress)rE   rr   
bytesSoFarbytesSinceLast)rR   rS   )rg   etag)rz   
partNumber)rr   parts)r;   complete)rE   rr   )r;   rH   r=   rI   rJ   rK   rL   rB   r2   rO   rk   poplistvalues	enumeraterP   requestsputraise_for_statusappendheaderspost)r(   init_msgr9   rr   rQ   completion_urlru   rv   presigned_urlsr|   ipresigned_urlrg   rr   r   r   r4      sj   

	


zLfsUploadCommand.runNr7   r   r   r   r   r      r8   r   )r#   r;   loggingr.   r3   r=   argparser   
contextlibr   typingr   r   r   r   huggingface_hub.commandsr   huggingface_hub.lfsr   	getLoggerr    rM   r	   r   rB   rO   rP   r   r   r   r   r   <module>   s&    
$%