Ë
    -Œ:jq  ã                   ó
  — d Z ddlZddlZddlZddlZddlZddlmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZ erddlm Z m!Z!  ejD                  e#«      Z$dZ%g d¢Z&dZ'dZ(dZ)dZ* e+«       Z,dZ-dZ.dZ/e-fde0de0de1de2fd„Z3de0de2fd„Z4 G d„ d«      Z5 G d„ d«      Z6 G d „ d!«      Z7 G d"„ d#«      Z8dddd$dd%œd&d'd(e2d)e2d*e9e   d+e9e   d,e2d-e2dz  d.e2e:z  dz  d/e2dz  d0e:d1e2dz  dd2fd3„Z;y)4ac  Streamed, multi-commit upload of a folder on top of the Xet upload protocol.

How it works:

- The **coordinator** (caller's thread) walks the list of files and asks the Hub, 256 files at a
  time, what each file is (regular git blob, xet file, ignored). Regular files are accumulated
  directly; xet files are registered into a `XetSession` upload-commit, which chunks, deduplicates,
  retries and uploads them in the background while the coordinator keeps going. No Python-side
  sha256 computation: `hf_xet` computes it during chunking (single read pass over each file).
- Whenever enough files have accumulated (adaptive batch size), the batch is handed over to the
  **committer** thread which joins the xet uploads, drops unchanged files, and creates a git
  commit for the batch. While a batch is being committed, the coordinator is already uploading
  the next one.
- Interrupted uploads are resumable by simply re-running the same call: already-committed files
  are dropped (no-op detection against the remote oid) and already-uploaded chunks are
  deduplicated by the xet storage backend, transferring ~0 bytes.
é    N)ÚTYPE_CHECKINGÚAnyÚCallable)Úquoteé   )Ú	constants)ÚCommitOperationAddÚCommitOperationDeleteÚ_fetch_upload_modesÚ_send_commitÚ_warn_on_overwriting_operations)ÚRepositoryNotFoundError)Úare_progress_bars_disabledÚlogging)ÚXetTokenTypeÚabort_xet_sessionÚget_xet_sessionÚxet_connection_info_refresh_urlÚxet_headers_without_auth)Ú
CommitInfoÚHfApié   )
é   é2   éK   éd   é}   éÈ   éú   i�  iX  éè  é   g      D@g     Àr@i  @r   ç      à?g      >@ÚcurrentÚtotalÚwidthÚreturnc                 óh   — |dk  rd|z  S t        t        | |z  d«      |z  «      }d|z  d||z
  z  z   S )Nr   u   â–‘ç      ð?u   â–ˆ)ÚintÚmin)r#   r$   r%   Úfilleds       úb/var/www/html/tokenscope/api/venv/lib/python3.12/site-packages/huggingface_hub/_upload_pipeline.pyÚ_barr-   U   sE   € Ø�‚zØ�u‰}ÐÜ”�W˜u‘_ cÓ*¨UÑ2Ó3€FØ�6‰>˜E U¨V¡^Ñ4Ñ4Ð4ó    Únc                 óŒ   — dD ]:  }t        | «      dk  r%| dk  r	| d›|› �c S | dk  r	| d›|› �c S | d›|› �c S | dz  } Œ< | d›d�S )	N)ÚBÚkBÚMBÚGBÚTBr    é
   z.2fr   ú.1fz.0fÚPB)Úabs)r/   Úunits     r,   Ú_format_bytesr;   \   sv   € Ø-ò ˆÜˆq‹6�DŠ=Ø�2ŠvØ˜C˜  Ð'Ò'Ø�S’Ø˜C˜  Ð'Ò'Ø˜�W˜T˜FÐ#Ò#Ø	ˆT‰	‰ðð �ˆW�Bˆ<Ðr.   c                   óð   — e Zd ZdZdZddededdfd„Zdd„Zdd	„Z	d
eddfd„Z
d
eddfd„Zd
eddfd„Zdee   ddfd„Zd
eddfd„Zdeddfd„Zdd„Zdd„Zdd„Zdd„Zdefd„Zdefd„Zdefd„Zdefd„Zy)Ú_LiveDisplayuR  Three-line live progress display on stderr::

        Preparing   â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆ  11,100 / 11,100 âœ“
        Uploading   â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–‘â–‘â–‘â–‘â–‘â–‘  580 / 603 files  3.8GB Â· 19.7MB/s
        Committing  â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–‘â–‘  10,800 / 11,100  14 commits

    A small renderer thread redraws the three lines in-place every ~0.5 s on a TTY
    (worker threads only update counters under a lock). When stderr is not a TTY,
    it falls back to a periodic ``logger.info`` summary instead.

    Disabling progress bars (e.g. agent output mode) only turns off the TTY renderer:
    the non-TTY log summaries are gated by the logger verbosity alone, so consumers
    tailing stderr during a long upload still see periodic progress.
    é   Útotal_filesÚenabledr&   Nc                 óè  — || _         |xr t        j                  j                  «       | _        | j                  xs# t
        j                  t        j                  «      | _	        t        j                  «       | _        d| _        t        j                  «       | _        d | _        d| _        d| _        d| _        t)        «       | _        d| _        d| _        d| _        d| _        d| _        d | _        y )NFr   ç        )Ú_totalÚsysÚstderrÚisattyÚ_ttyÚloggerÚisEnabledForr   ÚINFOÚ_activeÚ	threadingÚLockÚ_lockÚ_drawnÚEventÚ_stop_eventÚ_threadÚ	_preparedÚ_ignoredÚ
_xet_totalÚsetÚ	_xet_doneÚ
_committedÚ_nb_commitsÚ
_xet_bytesÚ
_speed_emaÚ_prev_bytesÚ
_prev_time)Úselfr?   r@   s      r,   Ú__init__z_LiveDisplay.__init__z   sº   € Ø!ˆŒØÒ3¤§
¡
× 1Ñ 1Ó 3ˆŒ	Ø—y‘yÒE¤F×$7Ñ$7¼¿¹Ó$EˆŒÜ—^‘^Ó%ˆŒ
ØˆŒÜ$Ÿ?™?Ó,ˆÔØ04ˆŒð ˆŒØˆŒØˆŒÜ#&£5ˆŒØˆŒØˆÔð ˆŒØˆŒØˆÔØ(,ˆ�r.   c                 óš  — | j                   sy | j                  rMt        j                  j	                  d| j
                  d›d�«       t        j                  j                  «        n$t        j                  d| j
                  d›d�«       t        j                  | j                  dd¬«      | _        | j                  j                  «        y )NzFound ú,z files to upload
z files to uploadzhf-upload-displayT©ÚtargetÚnameÚdaemon)rK   rG   rD   rE   ÚwriterC   ÚflushrH   ÚinforL   ÚThreadÚ_render_looprR   Ústart©r^   s    r,   rk   z_LiveDisplay.start“   s�   € Ø�|Š|ØØ�9Š9Ü�J‰J×Ñ˜v d§k¡k°! _Ð4FÐGÔHÜ�J‰J×ÑÕä�K‰K˜& §¡¨Q Ð/?Ð@ÔAÜ ×'Ñ'¨t×/@Ñ/@ÐGZÐcgÔhˆŒØ�‰×ÑÕr.   c                 ó   — | j                   �4| j                  j                  «        | j                   j                  «        | j                  r&| j
                  5  | j                  «        d d d «       y y # 1 sw Y   y xY w©N)rR   rQ   rV   ÚjoinrG   rN   Ú_redrawrl   s    r,   Úclosez_LiveDisplay.closež   sd   € Ø�<‰<Ð#Ø×Ñ× Ñ Ô"Ø�L‰L×ÑÔØ�9Š9Ø—‘ñ Ø—‘”÷ð ð ÷ð ús   ÁA4Á4A=r/   c                 óp   — | j                   5  | xj                  |z  c_        d d d «       y # 1 sw Y   y xY wrn   )rN   rS   ©r^   r/   s     r,   Únotify_preparedz_LiveDisplay.notify_prepared¨   s,   € Ø�Z‰Zñ 	 Ø�NŠN˜aÑ�N÷	 ÷ 	 ñ 	 úó   �,¬5c                 óp   — | j                   5  | xj                  |z  c_        d d d «       y # 1 sw Y   y xY wrn   )rN   rT   rs   s     r,   Únotify_ignoredz_LiveDisplay.notify_ignored¬   s,   € Ø�Z‰Zñ 	Ø�MŠM˜QÑ�M÷	÷ 	ñ 	úru   c                 óp   — | j                   5  | xj                  |z  c_        d d d «       y # 1 sw Y   y xY wrn   )rN   rU   rs   s     r,   Únotify_xet_registeredz"_LiveDisplay.notify_xet_registered°   ó,   € Ø�Z‰Zñ 	!Ø�OŠO˜qÑ �O÷	!÷ 	!ñ 	!úru   Únamesc                 ó|   — | j                   5  | j                  j                  |«       d d d «       y # 1 sw Y   y xY wrn   )rN   rW   Úupdate)r^   r{   s     r,   Únotify_xet_uploadedz _LiveDisplay.notify_xet_uploaded´   s0   € Ø�Z‰Zñ 	)Ø�N‰N×!Ñ! %Ô(÷	)÷ 	)ñ 	)ús   �2²;c                 óp   — | j                   5  | xj                  |z  c_        d d d «       y # 1 sw Y   y xY wrn   )rN   rX   rs   s     r,   Únotify_skippedz_LiveDisplay.notify_skipped¸   rz   ru   Ún_filesc                 óš   — | j                   5  | xj                  |z  c_        | xj                  dz  c_        d d d «       y # 1 sw Y   y xY w)Nr   )rN   rX   rY   )r^   r�   s     r,   Únotify_commitz_LiveDisplay.notify_commit¼   s=   € Ø�Z‰Zñ 	"Ø�OŠO˜wÑ&�OØ×Ò Ñ!Õ÷	"÷ 	"ñ 	"ús   �+AÁA
c                 óR   ‡ ‡— ‰ j                   sydŠdt        dt        ddfˆˆ fd„}|S )aS  Progress callback for one ``new_upload_commit``.

        The byte counters in ``group_report`` are cumulative *per upload-commit* and several
        upload-commits can be in flight at once (one finalizing, one filling), so each commit
        gets its own closure tracking its own previous value; increments are summed globally.
        Nr   Úgroup_reportÚitem_reportsr&   c                 ór  •— ‰j                   5  | j                  }‰xj                  t        d|‰z
  «      z  c_        |Š|j	                  «       D ]Q  }|j
                  dkD  sŒ|j                  |j
                  k(  sŒ-‰j                  j                  |j                  «       ŒS 	 d d d «       y # 1 sw Y   y xY w©Nr   )
rN   Útotal_transfer_bytes_completedrZ   ÚmaxÚvaluesÚtotal_bytesÚbytes_completedrW   ÚaddÚ	item_name)r…   r†   Ú	completedÚitemÚprevr^   s       €€r,   Úcallbackz/_LiveDisplay.new_xet_callback.<locals>.callbackÌ   s˜   ø€ à—‘ñ ;Ø(×GÑG�	Ø—’¤3 q¨)°dÑ*:Ó#;Ñ;•Ø �Ø(×/Ñ/Ó1ò ;�DØ×'Ñ'¨!Ó+°×0DÑ0DÈ×HXÑHXÓ0XØŸ™×*Ñ*¨4¯>©>Õ:ñ;÷	;÷ ;ñ ;ús   ŽAB-Á"B-Á<'B-Â-B6)rK   r   )r^   r“   r’   s   ` @r,   Únew_xet_callbackz_LiveDisplay.new_xet_callbackÁ   s5   ù€ ð �|Š|ØØˆð	;¤3ð 	;´cð 	;¸dö 	;ð ˆr.   c                 óÎ  — d}| j                   j                  t        «      s¸| j                  5  | j	                  «        | j
                  r| j                  «        nUt        j                  «       |z
  t        k\  r7t        j                  | j                  «       «       t        j                  «       }d d d «       | j                   j                  t        «      sŒ·y y # 1 sw Y   Œ+xY w)NrB   )rQ   ÚwaitÚ_REFRESH_INTERVALrN   Ú_update_speedrG   rp   ÚtimeÚ	monotonicÚ_NON_TTY_LOG_INTERVALrH   rh   Ú_summary)r^   Úlast_logs     r,   rj   z_LiveDisplay._render_loopÚ   sž   € ØˆØ×"Ñ"×'Ñ'Ô(9Ô:Ø—‘ñ 0Ø×"Ñ"Ô$Ø—9’9Ø—L‘L•NÜ—^‘^Ó%¨Ñ0Ô4IÒIÜ—K‘K §¡£Ô0Ü#Ÿ~™~Ó/�H÷0ð ×"Ñ"×'Ñ'Ô(9×:÷0ð 0ús   ®BCÃC$c                 ó:  — t        j                  «       }| j                  �c|| j                  kD  rT| j                  | j                  z
  || j                  z
  z  }| j
                  dk(  r|nd|z  d| j
                  z  z   | _        || _        | j                  | _        y )Nr   g333333Ó?gffffffæ?)r™   rš   r]   rZ   r\   r[   )r^   ÚnowÚrates      r,   r˜   z_LiveDisplay._update_speedå   s�   € Ü�n‰nÓˆØ�?‰?Ð&¨3°·±Ò+@Ø—O‘O d×&6Ñ&6Ñ6¸3ÀÇÁÑ;PÑQˆDØ&*§o¡o¸Ò&:™dÀÀdÁ
ÈSÐSW×SbÑSbÑMbÑ@bˆDŒOØˆŒØŸ?™?ˆÕr.   c                 óæ  — | j                   r-t        j                  j                  d| j                  › d�«       t        j                  «       j                  }| j                  «       | j                  «       | j                  «       fD ]C  }t        |«      |dz
  kD  r|d |dz
   dz   n|}t        j                  j                  d|› d�«       ŒE t        j                  j                  «        d| _         y )	Nz[ÚAr   é   z...z[Kú
T)rO   rD   rE   rf   Ú_N_LINESÚshutilÚget_terminal_sizeÚcolumnsÚ_line_preparingÚ_line_uploadingÚ_line_committingÚlenrg   )r^   r%   ÚlineÚ	truncateds       r,   rp   z_LiveDisplay._redrawí   sÈ   € Ø�;Š;Ü�J‰J×Ñ˜u T§]¡] O°1Ð5Ô6Ü×(Ñ(Ó*×2Ñ2ˆØ×)Ñ)Ó+¨T×-AÑ-AÓ-CÀT×EZÑEZÓE\Ð]ò 	7ˆDÜ58¸³YÀÈÁÒ5J˜˜[˜u q™yÐ)¨EÒ1ÐPTˆIÜ�J‰J×Ñ˜x¨	 {°"Ð5Õ6ð	7ô 	�
‰
×ÑÔØˆ�r.   c                 ó¾   — | j                   | j                  k\  rdnd}dt        | j                   | j                  «      › d| j                   d›d| j                  d›|› �S )Nõ    âœ“Ú z  Preparing   ú  ra   ú / )rS   rC   r-   )r^   Údones     r,   r©   z_LiveDisplay._line_preparing÷   s[   € ØŸ™¨4¯;©;Ò6‰v¸BˆØ¤ T§^¡^°T·[±[Ó AÐBÀ"ÀTÇ^Á^ÐTUÐDVÐVYÐZ^×ZeÑZeÐfgÐYhÐimÐhnÐoÐor.   c           	      óv  — | j                   dk(  r7| j                  | j                  k\  rt        dd«      nt        dd«      }d|› d�S t	        | j
                  «      }g }| j                  dkD  r$|j                  t        | j                  «      «       | j                  dkD  r'|j                  t        | j                  «      › d�«       |rddj                  |«      › �nd}| j                  | j                  k\  r|| j                   k\  rd	nd}dt        || j                   «      › d|d
›d| j                   d
›d|› |› �	S )Nr   r   z  Uploading   z  -z/sr²   u    Â· r±   r°   ra   r³   z files)rU   rS   rC   r-   r¬   rW   rZ   Úappendr;   r[   ro   )r^   ÚbarÚn_doneÚpartsÚextrar´   s         r,   rª   z_LiveDisplay._line_uploadingû   s  € Ø�?‰?˜aÒØ $§¡°$·+±+Ò =”$�q˜!”*Ä4ÈÈ1Ã:ˆCØ# C 5¨Ð,Ð,Ü�T—^‘^Ó$ˆØˆØ�?‰?˜QÒØ�L‰Lœ t§¡Ó7Ô8Ø�?‰?˜QÒØ�L‰LœM¨$¯/©/Ó:Ð;¸2Ð>Ô?Ù-2�"�V—[‘[ Ó'Ð(Ñ)¸ˆØŸ™¨4¯;©;Ò6¸6ÀTÇ_Á_Ò;T‰vÐZ\ˆØ¤ V¨T¯_©_Ó =Ð>¸bÀÈÀ
È#ÈdÏoÉoÐ^_ÐM`Ð`fÐglÐfmÐnrÐmsÐtÐtr.   c                 ó  — | j                   | j                  z
  }| j                  dkD  rd| j                  › d�nd}| j                  |cxk\  rdkD  rn ndnd}dt	        | j                  |«      › d| j                  d›d	|d›|› |› �S )
Nr   r²   z commitsr±   r   r°   z  Committing  ra   r³   )rC   rT   rY   rX   r-   )r^   Ú	effectiveÚcommits_strr´   s       r,   r«   z_LiveDisplay._line_committing	  s‘   € Ø—K‘K $§-¡-Ñ/ˆ	Ø9=×9IÑ9IÈAÒ9M˜˜4×+Ñ+Ð,¨HÑ5ÐSUˆØŸ™¨IÔ9¸Õ9‰v¸rˆàœT $§/¡/°9Ó=Ð>¸bÀÇÁÐQRÐ@SÐSVÐW`ÐabÐVcÐdoÐcpÐquÐpvÐwð	
r.   c                 óì   — d| j                   d›d| j                  d›dt        | j                  «      d›d| j                  d›dt        | j                  «      › d| j                  d›d| j                  › d�S )	NzUploading... ra   ú/z files checked, z uploaded (z transferred), z committed in z
 commit(s))	rS   rC   r¬   rW   rU   r;   rZ   rX   rY   rl   s    r,   rœ   z_LiveDisplay._summary  sy   € à˜DŸN™N¨1Ð-¨Q¨t¯{©{¸1¨oÐ=MÜ�4—>‘>Ó" 1Ð% Q t§¡°qÐ&9¸Ä]ÐSW×SbÑSbÓEcÐDdÐdsØ�‰˜qÐ! °×0@Ñ0@Ð/AÀðMð	
r.   )T©r&   N)r&   zCallable | None)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r¥   r)   Úboolr_   rk   rq   rt   rw   ry   ÚlistÚstrr~   r€   rƒ   r”   rj   r˜   rp   r©   rª   r«   rœ   © r.   r,   r=   r=   h   sð   „ ñð €Hñ- Cð -°$ð -À$ó -ó2	óð  ð  ¨ó  ð ð ¨ó ð! sð !¨tó !ð)¨¨c©ð )°tó )ð! ð !¨ó !ð" Sð "¨Tó "ó
ó2	0ó+óðp ó pðu ó uð
 #ó 
ð
˜#ô 
r.   r=   c                   óJ   — e Zd ZdZd
d„Zedefd„«       Zdededdfd„Z	d
d	„Z
y)Ú_CommitPacerzOAdaptive number of files per commit, to stay below server-side commit timeouts.r&   Nc                 ó   — t         | _        y rn   )ÚINITIAL_COMMIT_SIZE_INDEXÚ_indexrl   s    r,   r_   z_CommitPacer.__init__  s	   € Ü/ˆ�r.   c                 ó(   — t         | j                     S rn   )ÚCOMMIT_SIZE_SCALErÍ   rl   s    r,   rc   z_CommitPacer.target  s   € ä  §¡Ñ-Ð-r.   ÚdurationÚnb_filesc                 óâ   — |t         k  r>|| j                  k\  r/t        | j                  dz   t	        t
        «      dz
  «      | _        y |t         kD  rt        | j                  dz
  d«      | _        y y ©Nr   r   )ÚTARGET_COMMIT_DURATIONrc   r*   rÍ   r¬   rÏ   rŠ   )r^   rÐ   rÑ   s      r,   Úrecord_successz_CommitPacer.record_success#  s[   € ØÔ,Ò,°¸T¿[¹[Ò1HÜ˜dŸk™k¨A™o¬sÔ3DÓ/EÈÑ/IÓJˆD�KØÔ.Ò.Ü˜dŸk™k¨A™o¨qÓ1ˆD�Kð /r.   c                 ó@   — t        | j                  dz
  d«      | _        y rÓ   )rŠ   rÍ   rl   s    r,   Úrecord_failurez_CommitPacer.record_failure)  s   € Ü˜$Ÿ+™+¨™/¨1Ó-ˆ�r.   rÀ   )rÁ   rÂ   rÃ   rÄ   r_   Úpropertyr)   rc   ÚfloatrÕ   r×   rÈ   r.   r,   rÊ   rÊ     sD   „ ÙYó0ð ð.˜ò .ó ð.ð2 uð 2¸ð 2Àó 2ô.r.   rÊ   c                   ó   — e Zd ZdZdd„Zy)Ú_BatchzSA group of files destined to a single git commit, with their in-flight xet uploads.Nc                 ón   — g | _         d| _        d | _        g | _        t	        j
                  «       | _        y rˆ   )ÚopsÚregular_bytesÚ
xet_commitÚhandlesr™   rš   Ú
created_atrl   s    r,   r_   z_Batch.__init__0  s-   € Ø-/ˆŒØ"#ˆÔØ#ˆŒØ=?ˆŒÜ!%§¡Ó!1ˆ�r.   rÀ   )rÁ   rÂ   rÃ   rÄ   r_   rÈ   r.   r,   rÛ   rÛ   -  s
   „ Ù]ô2r.   rÛ   c                   óê   — e Zd Zdddededee   dee   deded	z  d
eez  d	z  ded	z  deded	z  dd	fd„Zdd„Z	dd„Z
dedd	fd„Zdedd	fd„Zdd„Zdedd	fd„Zdee   dd	fd„Zdee   dd	fd„Zdd„Zy	)Ú_UploadPipelineÚapir   Úrepo_idÚ	repo_typeÚadd_operationsÚdelete_operationsÚcommit_messageÚcommit_descriptionNÚtokenÚrevisionÚ	create_prÚparent_commitr&   c       
         ó  — || _         || _        || _        || _        || _        || _        || _        || _        |j                  |¬«      | _	        |	xs t        j                  | _        |
| _        || _        t        | j                  d¬«      | _        | j                   | _        d | _        d | _        d| _        d | _        t-        «       | _        t1        j2                  d¬«      | _        g | _        t9        j:                  «       | _        t?        tA        |«      tC        «        ¬«      | _"        tG        tH        jJ                  ||| j                   |jL                  ¬«      }|
r|d	z  }tO        «       | _(        || j                  tS        | j                  «      d
œ| _*        d | _+        |D ]O  }|jX                  dk(  sŒ|j[                  «       5 }|j]                  «       j_                  «       | _+        d d d «        y  y # 1 sw Y    y xY w)N)rë   r±   ©Úsafer   r   )Úmaxsize)r?   r@   )Ú
token_typerå   ræ   rì   Úendpointz?create_pr=1)Útoken_refresh_urlÚtoken_refresh_headersÚcustom_headersz
.gitignore)0rä   rå   ræ   rç   rè   ré   rê   rë   Ú_build_hf_headersÚheadersr   ÚDEFAULT_REVISIONrì   rí   rî   r   Úbase_revision_quotedÚcommit_revision_quotedÚpr_urlÚpr_revisionÚ
nb_commitsÚlast_commit_inforÊ   ÚpacerÚqueueÚQueueÚbatch_queueÚerrorsrL   rP   Úabort_eventr=   r¬   r   Údisplayr   r   ÚWRITErô   r   Úxet_sessionr   Úxet_commit_kwargsÚgitignore_contentÚpath_in_repoÚas_fileÚreadÚdecode)r^   rä   rå   ræ   rç   rè   ré   rê   rë   rì   rí   rî   Úrefresh_urlÚopÚfs                  r,   r_   z_UploadPipeline.__init__9  sÅ  € ð ˆŒØˆŒØ"ˆŒØ,ˆÔØ!2ˆÔØ,ˆÔØ"4ˆÔØˆŒ
Ø×,Ñ,°5Ð,Ó9ˆŒØ Ò>¤I×$>Ñ$>ˆŒØ"ˆŒØ*ˆÔô
 %*¨$¯-©-¸bÔ$AˆÔ!ð '+×&?Ñ&?ˆÔ#Ø"&ˆŒØ'+ˆÔØˆŒØ59ˆÔÜ!“^ˆŒ
ô ).¯©¸AÔ(>ˆÔØ+-ˆŒÜ$Ÿ?™?Ó,ˆÔÜ#´°NÓ0CÔQkÓQmÐMmÔnˆŒô 6Ü#×)Ñ)ØØØ×.Ñ.Ø—\‘\ô
ˆñ Ø˜>Ñ)ˆKÜ*Ó,ˆÔà!,Ø%)§\¡\Ü6°t·|±|ÓDñ"
ˆÔð .2ˆÔØ ò 	ˆBØ�‰ ,Ó.Ø—Z‘Z“\ð ? QØ-.¯V©V«X¯_©_Ó->�DÔ*÷?áñ		÷?áús   Ç
$G:Ç:H	c           	      óþ  — t        g | j                  ¢| j                  ¢«       t        j                  | j
                  dd¬«      }|j                  «        | j                  j                  «        	 | j                  «        	 | j                  j                  «       r|j                  d¬«       n/| j                  j!                  t"        «       |j                  «        | j                  j%                  «        | j                  j                  «       r=| j&                  �1t(        j+                  d| j,                  › d| j&                  › d�«       	 | j.                  r| j.                  d	   ‚| j1                  «       S # t        $ r& | j                  j                  «        t        «        ‚ w xY w# | j                  j                  «       r|j                  d¬«       n/| j                  j!                  t"        «       |j                  «        | j                  j%                  «        | j                  j                  «       r>| j&                  �1t(        j+                  d| j,                  › d| j&                  › d�«       w w w xY w)
Nzhf-upload-committerTrb   r6   ©ÚtimeoutzUpload to pull request zF did not complete. To resume into the same PR, re-run with `revision="z^"` (without `create_pr=True`). Re-running with `create_pr=True` would open a new pull request.r   )r   rè   rç   rL   ri   Ú_committer_looprk   r  Ú_coordinator_loopÚBaseExceptionr  rV   r   Úis_setro   r  ÚputÚ	_SENTINELrq   rþ   rH   Úwarningrý   r  Ú_final_commit_info)r^   Ú	committers     r,   Úrunz_UploadPipeline.run‚  s  € Ü'Ð(W¨$×*@Ñ*@Ð(WÀ4×CVÑCVÐ(WÔXÜ×$Ñ$¨D×,@Ñ,@ÐG\ÐeiÔjˆ	Ø�‰ÔØ�‰×ÑÔð	Ø×"Ñ"Õ$ð ×Ñ×&Ñ&Ô(ð —‘ r�Õ*à× Ñ ×$Ñ$¤YÔ/Ø—‘Ô Ø�L‰L×ÑÔ Ø×Ñ×&Ñ&Ô(¨T×-=Ñ-=Ð-IÜ—‘Ø-¨d¯k©k¨]ð ;8Ø8<×8HÑ8HÐ7Ið JLðLõð
 �;Š;Ø—+‘+˜a‘.Ð Ø×&Ñ&Ó(Ð(øô- ò 	Ø×Ñ× Ñ Ô"ÜÔØð	ûð
 ×Ñ×&Ñ&Ô(ð —‘ r�Õ*à× Ñ ×$Ñ$¤YÔ/Ø—‘Ô Ø�L‰L×ÑÔ Ø×Ñ×&Ñ&Ô(¨T×-=Ñ-=Ð-IÜ—‘Ø-¨d¯k©k¨]ð ;8Ø8<×8HÑ8HÐ7Ið JLðLõð .JÐ(ús   Á1E: Å:/F)Æ)F, Æ,CI<c                 ó¦  — dd l }t        «       }t        dt        | j                  «      t
        «      D �]a  }| j                  j                  «       r| j                  |«        y | j                  ||t
        z    }	 t        || j                  | j                  | j                  | j                  | j                  j                  | j                   | j"                  ¬«       | j,                  j/                  t        |«      «       |D �]�  }|j0                  r?t2        j5                  d|j6                  › d�«       | j,                  j9                  d«       ŒO|j:                  dk(  r+|xj<                  |j>                  j@                  z  c_        �nL|jB                  €E | jD                  jF                  dd| j,                  jI                  «       i| jJ                  ¤Ž|_!        |j>                  jL                  r$|j>                  jN                  jQ                  «       n|jR                  }tU        |jV                  tX        «      r3|jB                  j[                  |jV                  ||j6                  ¬	«      }	n0|jB                  j]                  t_        |jV                  «      |¬
«      }	|j`                  jc                  ||	f«       | j,                  je                  d«       |jf                  jc                  |«       t        |jf                  «      | jh                  jj                  k\  sW|j<                  tl        k\  sDto        jp                  «       |jr                  z
  tt        kD  s�ŒYt        |jf                  «      dkD  s�Œs| jw                  |«       t        «       }�Œ� �Œd | jw                  |«       y # t$        $ r}ddlm} |j+                  |«       ‚ d }~ww xY w)Nr   )Ú	additionsræ   rå   rù   rì   rô   rí   r  r   )Ú$_CREATE_COMMIT_NO_REPO_ERROR_MESSAGEzSkipping upload for 'z' (ignored by gitignore rules).ÚregularÚprogress_callback)Úsha256rd   )r%  rÈ   )<Úhf_xetrÛ   Úranger¬   rç   ÚPREUPLOAD_BATCH_SIZEr  r  Ú_abort_batchr   ræ   rå   rù   rû   rä   rô   rí   r  r   Úhf_apir"  Úappend_to_messager  rt   Ú_should_ignorerH   Údebugr  rw   Ú_upload_moderÞ   Úupload_infoÚsizerß   r	  Únew_upload_commitr”   r
  Ú	is_hashedr%  ÚhexÚCOMPUTE_SHA256Ú
isinstanceÚpath_or_fileobjÚbytesÚstart_upload_bytesÚstart_upload_filerÇ   rà   r¶   ry   rÝ   r  rc   ÚREGULAR_CONTENT_BYTES_BUDGETr™   rš   rá   ÚMAX_COMMIT_INTERVALÚ_enqueue)
r^   r&  Úbatchrk   ÚchunkÚer"  r  Úsha_argÚhandles
             r,   r  z!_UploadPipeline._coordinator_loop£  sý  € Ûä“ˆÜ˜1œc $×"5Ñ"5Ó6Ô8LÓMó 5	%ˆEØ×Ñ×&Ñ&Ô(Ø×!Ñ! %Ô(ÙØ×'Ñ'¨°Ô8LÑ0LÐMˆEðÜ#Ø#Ø"Ÿn™nØ ŸL™LØ ŸL™LØ!×6Ñ6Ø!ŸX™X×.Ñ.Ø"Ÿn™nØ&*×&<Ñ&<õ	ð �L‰L×(Ñ(¬¨U«Ô4Øó %�Ø×$Ò$Ü—L‘LÐ#8¸¿¹Ð8IÐIhÐ!iÔjØ—L‘L×/Ñ/°Ô2ØØ—?‘? iÒ/Ø×'Ò'¨2¯>©>×+>Ñ+>Ñ>×'à×'Ñ'Ð/Ø+M¨4×+;Ñ+;×+MÑ+Mñ ,Ø.2¯l©l×.KÑ.KÓ.Mð,ØQU×QgÑQgñ,˜Ô(ð
 >@¿^¹^×=UÒ=U˜bŸn™n×3Ñ3×7Ñ7Ô9Ð[a×[pÑ[p�GÜ! "×"4Ñ"4´eÔ<Ø!&×!1Ñ!1×!DÑ!DØ×.Ñ.°wÀRÇ_Á_ð "Eó "™ð "'×!1Ñ!1×!CÑ!CÄCÈ×HZÑHZÓD[ÐdkÐ!CÓ!l˜Ø—M‘M×(Ñ(¨"¨f¨Ô6Ø—L‘L×6Ñ6°qÔ9Ø—	‘	× Ñ  Ô$ô ˜Ÿ	™	“N d§j¡j×&7Ñ&7Ò7Ø×*Ñ*Ô.JÒJÜŸ™Ó(¨5×+;Ñ+;Ñ;Ô>QÔQÔVYÐZ_×ZcÑZcÓVdÐghÔVhà—M‘M %Ô(Ü"›H’Eò?%ð-5	%ðl 	�‰�eÕøôM +ò ÝHà×#Ñ#Ð$HÔIØûð	ús   Á;A#N*Î*	OÎ3OÏOr=  c                 óJ  — t        |j                  «      dk(  r(| j                  dk(  rt        | j                  «      dkD  sy | j                  j                  «       s	 | j                  j                  |d¬«       y | j                  |«       y # t        j                  $ r Y Œ`w xY w)Nr   r(   r  )r¬   rÝ   rÿ   rè   r  r  r  r  r  ÚFullr)  ©r^   r=  s     r,   r<  z_UploadPipeline._enqueueß  sŽ   € Üˆu�y‰y‹>˜QÒ¨¯©¸1Ò(<ÄÀT×E[ÑE[ÓA\Ð_`ÒA`Øà×"Ñ"×)Ñ)Ô+ðØ× Ñ ×$Ñ$ U°CÐ$Ô8Øð 	×Ñ˜%Õ øô —:‘:ò Ùðús   ÁB ÂB"Â!B"c                 ór   — |j                   �	 |j                   j                  «        y y # t        $ r Y y w xY wrn   )rß   ÚabortÚ	ExceptionrD  s     r,   r)  z_UploadPipeline._abort_batchë  s>   € Ø×ÑÐ'ðØ× Ñ ×&Ñ&Õ(ð (øô ò Ùðús   Ž* ª	6µ6c                 óð  — 	 	 | j                   j                  d¬«      }|t        u ry 	 | j                  j                  «       s| j                  |«       n| j                  |«       Œe# t        j                  $ r | j                  j                  «       rY y Y Œ—w xY w# t        $ rP}| j                  |«       | j                  j                  |«       | j                  j                  «        Y d }~ŒŠd }~ww xY w)Nr"   r  )r  Úgetr  ÚEmptyr  r  r  Ú_process_batchr)  r  r  r¶   rV   )r^   r=  r?  s      r,   r  z_UploadPipeline._committer_loopô  sÛ   € ØðØ×(Ñ(×,Ñ,°SÐ,Ó9�ð
 œ	Ñ!Øð'Ø×'Ñ'×.Ñ.Ô0Ø×'Ñ'¨Õ.à×%Ñ% eÔ,ð øô —;‘;ò Ø×#Ñ#×*Ñ*Ô,ÙÙðûô !ò 'Ø×!Ñ! %Ô(Ø—‘×"Ñ" 1Ô%Ø× Ñ ×$Ñ$×&Ñ&ûð'ús)   ƒA' ©=B Á'-BÂBÂ	C5Â%AC0Ã0C5c           
      óÎ  — |j                   ��|j                   j                  «        |j                  D ]h  \  }}|j                  j                  sFt
        j                  |j                  «       j                  j                  «      |j                  _	        d|_
        Œj | j                  j                  |j                  D ��cg c]@  \  }}t        |j                  t
        «      st        |j                  «      n|j                   ‘ŒB c}}«       g }|j"                  D ]w  }|j$                  �X|j$                  |j&                  k(  r?t(        j+                  d|j                   › d�«       | j                  j-                  d«       Œg|j/                  |«       Œy t1        |«      dkD  s'| j2                  dk(  r+t1        | j4                  «      dkD  r| j7                  |«       y y y c c}}w )NTzSkipping commit for 'z' (file unchanged).r   r   )rß   Úwait_to_finishrà   r/  r2  r7  ÚfromhexÚresultÚxet_infor%  Ú_is_uploadedr  r~   r5  r6  rÇ   r  rÝ   Ú_remote_oidÚ
_local_oidrH   r-  r€   r¶   r¬   rÿ   rè   Ú_commit_with_split)r^   r=  r  rA  Ú_Úops_to_commits         r,   rK  z_UploadPipeline._process_batch  s‹  € ð ×ÑÑ'Ø×Ñ×+Ñ+Ô-Ø#Ÿm™mò '‘
��FØ—~‘~×/Ò/Ü,1¯M©M¸&¿-¹-»/×:RÑ:R×:YÑ:YÓ,Z�B—N‘NÔ)Ø"&�•ð'ð
 �L‰L×,Ñ,ð "'§¡÷á˜˜Aô 4>¸b×>PÑ>PÔRWÔ3X”C˜×*Ñ*Ô+Ð^`×^mÑ^mÑmóôð ˆØ—)‘)ò 	%ˆBØ�~‰~Ð)¨b¯n©nÀÇÁÒ.MÜ—‘Ð4°R·_±_Ð4EÐEXÐYÔZØ—‘×+Ñ+¨AÔ.ØØ× Ñ  Õ$ð	%ô ˆ}Ó Ò! d§o¡o¸Ò&:¼sÀ4×CYÑCYÓ?ZÐ]^Ò?^Ø×#Ñ# MÕ2ð @_Ð&:ùó#s   ÃAG!
rÝ   c                 óŽ  — 	 | j                  |«       y # t        $ r§}| j                  j                  «        t	        |«      t
        d   k  r‚ t        j                  dt	        |«      › d|›d�«       | j                  j                  }t        dt	        |«      |«      D ]  }| j                  ||||z    «       Œ Y d }~y d }~ww xY w)Nr   z
Commit of z files failed (z). Retrying in smaller chunks.)Ú
_do_commitrG  r  r×   r¬   rÏ   rH   r  rc   r'  rT  )r^   rÝ   r?  rc   rk   s        r,   rT  z"_UploadPipeline._commit_with_split'  s±   € ð		EØ�O‰O˜CÕ øÜò 	EØ�J‰J×%Ñ%Ô'Ü�3‹xÔ,¨QÑ/Ò/ØÜ�N‰N˜Z¬¨C« z°ÀÀÐEcÐdÔeØ—Z‘Z×&Ñ&ˆFÜ˜q¤# c£(¨FÓ3ò E�Ø×'Ñ'¨¨E°E¸F±NÐ(CÕDõEûð	Eús   ‚ ”	C�BB?Â?Cc                 ó   — | j                   r³| j                  €§| j                  j                  | j                  | j
                  | j                  | j                  | j                  ¬«      }|j                  €t        d«      ‚|j                  | _        |j                  | _        t        |j                  d¬«      | _        t        |«      }| j                   dk(  rt        | j"                  «      |z   }| j                   dk(  r| j
                  n| j
                  › d| j                   dz   › d�}t%        j&                  «       }t)        |i || j                  xs d| j                  | j                  | j*                  | j                  | j                  j,                  | j                   dk(  r| j.                  nd d	¬
«      | _        t%        j&                  «       |z
  }| j2                  j5                  |t7        |«      «       | xj                   dz  c_        |D ]	  }d	|_        Œ | j:                  j=                  t7        |«      «       t>        jA                  dt7        |«      › d|d›d| j0                  jB                  › �«       y )N)rå   Útitlerë   Údescriptionræ   zCServer did not return a git reference for the created pull request.r±   rð   r   z (part r   ú)T)Ú
operationsÚfiles_to_copyré   rê   ræ   rå   rù   rì   rô   rî   Úretry_on_errorz
Committed z file(s) in r7   zs: )"rí   rþ   rä   Úcreate_pull_requestrå   ré   rë   rê   ræ   Úgit_referenceÚ
ValueErrorÚurlrý   r   rü   rÆ   rÿ   rè   r™   rš   r   rù   rô   rî   r   r  rÕ   r¬   Ú_is_committedr  rƒ   rH   r-  Ú
commit_url)r^   rÝ   Úprr]  ré   Út0rÐ   r  s           r,   rX  z_UploadPipeline._do_commit3  s  € Ø�>Š>˜d×.Ñ.Ð6ð —‘×-Ñ-ØŸ™Ø×)Ñ)Ø—j‘jØ ×3Ñ3ØŸ.™.ð .ó ˆBð ×ÑÐ'Ü Ð!fÓgÐgØŸ&™&ˆDŒKØ!×/Ñ/ˆDÔÜ*/°×0@Ñ0@ÀrÔ*JˆDÔ'ä $ S£	ˆ
Ø�?‰?˜aÒä˜d×4Ñ4Ó5¸
ÑBˆJð $(§?¡?°aÒ#7ˆD×ÒÀ×@SÑ@SÐ?TÐT[Ð\`×\kÑ\kÐnoÑ\oÐ[pÐpqÐ=rð 	ô �^‰^Óˆô !-Ø!ØØ)Ø#×6Ñ6Ò<¸"Ø—n‘nØ—L‘LØ—L‘LØ×0Ñ0Ø—X‘X×&Ñ&Ø04·±À1Ò0D˜$×,Ò,È$Øô!
ˆÔô —>‘>Ó# bÑ(ˆØ�
‰
×!Ñ! (¬C°«HÔ5Ø�Š˜1Ñ�àò 	$ˆBØ#ˆBÕð	$à�‰×"Ñ"¤3 s£8Ô,Ü�‰�z¤# c£( ¨<¸À°~ÀSÈ×I^ÑI^×IiÑIiÐHjÐkÕlr.   c                 óœ  — ddl m} | j                  €õt        j	                  d«       | j
                  j                  | j                  | j                  | j                  ¬«      }| j
                  j                  }| j                  t        j                  k7  r|› d| j                  › d�} ||› d| j                  › d|j                  › �| j                  | j                  xs d|j                  | j
                  j                  ¬	«      S | j                   dkD  r#t        j#                  d
| j                   › d�«       | j$                  �| || j                  j&                  | j                  j                  | j                  j                  | j                  j(                  | j$                  | j
                  j                  ¬«      S | j                  S )Nr   )r   zPNo files have been modified since last commit. Skipping to prevent empty commit.)rå   ræ   rì   r¿   Úsz/commit/r±   )re  ré   rê   ÚoidÚ	_endpointzUpload completed in z	 commits.)re  ré   rê   rj  rý   rk  )r*  r   r   rH   r  rä   Ú	repo_inforå   ræ   rì   rô   r   ÚREPO_TYPE_MODELÚsharé   rê   rÿ   rh   rý   re  rj  )r^   r   rh   Ú
url_prefixs       r,   r  z"_UploadPipeline._final_commit_infoj  st  € Ý&à× Ñ Ð(ô �N‰NÐmÔnØ—8‘8×%Ñ%¨d¯l©lÀdÇnÁnÐ_c×_lÑ_lÐ%ÓmˆDØŸ™×*Ñ*ˆJØ�~‰~¤×!:Ñ!:Ò:Ø *˜|¨1¨T¯^©^Ð,<¸AÐ>�
ÙØ(˜\¨¨4¯<©<¨.¸ÀÇÁÀ
ÐKØ#×2Ñ2Ø#'×#:Ñ#:Ò#@¸bØ—H‘HØŸ(™(×+Ñ+ôð ð �?‰?˜QÒÜ�K‰KÐ.¨t¯©Ð.?¸yÐIÔJØ�;‰;Ð"áØ×0Ñ0×;Ñ;Ø#×4Ñ4×CÑCØ#'×#8Ñ#8×#KÑ#KØ×)Ñ)×-Ñ-Ø—{‘{ØŸ(™(×+Ñ+ôð ð ×$Ñ$Ð$r.   )r&   r   rÀ   )rÁ   rÂ   rÃ   rÇ   rÆ   r	   r
   rÅ   r_   r  r  rÛ   r<  r)  r  rK  rT  rX  r  rÈ   r.   r,   rã   rã   8  s4  „ ðGàðGð ð	Gð
 ðGð Ð/Ñ0ðGð  Ð 5Ñ6ðGð ðGð   $™JðGð �T‰z˜DÑ ðGð ˜‘*ðGð ðGð ˜T‘zðGð 
óGóR)óB:ðx
!˜fð 
!¨ó 
!ð &ð ¨Tó ó'ð(3 Fð 3¨tó 3ð>
E dÐ+=Ñ&>ð 
EÀ4ó 
Eð3m˜dÐ#5Ñ6ð 3m¸4ó 3môn%r.   rã   F)rê   rë   rì   rí   rî   rä   r   rå   ræ   rç   rè   ré   rê   rë   rì   rí   rî   r   c       
         óJ   — t        | |||||||||	|
¬«      j                  «       S )z¨Upload a prepared list of operations through the streamed multi-commit pipeline.

    Requires `hf_xet` to be installed. See module docstring for the architecture.
    )
rå   ræ   rç   rè   ré   rê   rë   rì   rí   rî   )rã   r  )rä   rå   ræ   rç   rè   ré   rê   rë   rì   rí   rî   s              r,   Úpipelined_uploadrq  ‹  s;   € ô& ØØØØ%Ø+Ø%Ø-ØØØØ#ô÷ 
�cƒeðr.   )<rÄ   r  r¦   rD   rL   r™   Útypingr   r   r   Úurllib.parser   r±   r   Ú_commit_apir	   r
   r   r   r   r  r   Úutilsr   r   Ú
utils._xetr   r   r   r   r   r*  r   r   Ú
get_loggerrÁ   rH   r(  rÏ   rÌ   rÔ   r;  r:  Úobjectr  Ú
_BAR_WIDTHr—   r›   rÙ   r)   rÇ   r-   r;   r=   rÊ   rÛ   rã   rÆ   rÅ   rq  rÈ   r.   r,   ú<module>rz     s¶  ðñó$ Û Û 
Û Û ß /Ñ /Ý å ÷õ õ ,ß 6÷õ ñ ß)à	ˆ×	Ñ	˜HÓ	%€ð Ð ò EÐ ØÐ ØÐ ØÐ ð  1Ð á‹H€	ð €
ØÐ ØÐ ð 5?ñ 5�%ð 5 ð 5¨cð 5À3ó 5ð	�Uð 	˜só 	÷n
ñ n
÷b.ñ .÷(2ñ 2÷P%ñ P%ðv
 &*Ø#ØØØ $òØ	ðð ðð ð	ð
 Ð+Ñ,ðð Ð1Ñ2ðð ðð ˜d™
ðð �‰:˜Ñðð �D‰jðð ðð ˜‘:ðð ôr.   