+
    &jD                         ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
 ^ RIt^RIHtHt ^RIHtHt ] P$                  ! R4      t ! R R	]4      t ! R
 R]4      tR# )    N)suppress)datetime)quote)AbstractBufferedFileAbstractFileSystem)infer_storage_optionstokenizewebhdfsc                   R  a a ] tR t^t oRt]! ]P                  ! 4       4      tRt	R V 3R llt
]R 4       tR tR!R ltR"R lt]R 4       t]R 4       t]R	 4       tR
 tR tR tR#R ltR tR tR tR$R ltR tR tR tR%R lt R t!R t"R#R lt#R t$R#R lt%R t&R t'R t(Rt)Vt*V ;t+# )&WebHDFSa  
Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways.

Four auth mechanisms are supported:

insecure: no auth is done, and the user is assumed to be whoever they
    say they are (parameter ``user``), or a predefined value such as
    "dr.who" if not given
spnego: when kerberos authentication is enabled, auth is negotiated by
    requests_kerberos https://github.com/requests/requests-kerberos .
    This establishes a session based on existing kinit login and/or
    specified principal/password; parameters are passed with ``kerb_kwargs``
token: uses an existing Hadoop delegation token from another secured
    service. Indeed, this client can also generate such tokens when
    not insecure. Note that tokens expire, but can be renewed (by a
    previously specified user) and may allow for proxying.
basic-auth: used when both parameter ``user`` and parameter ``password``
    are provided.

c                @  < V P                   '       d   R# \        SV `  ! R/ VB  V
'       d   RMR RV RV R2V n        W0n        T;'       g    / V n        / V n        T	;'       g    / V n        Ve"   Vf   Ve   \        R4      hW@P                  R&   WPn	        W`n
        Ve   Vf   \        R	4      hMVe   WPP                  R
&   Ve   WpP                  R&   V'       d   Ve   \        R4      hWn        Wn        V P                  4        R\        W4       2V n        R# )ab  
Parameters
----------
host: str
    Name-node address
port: int
    Port for webHDFS
kerberos: bool
    Whether to authenticate with kerberos for this connection
token: str or None
    If given, use this token on every call to authenticate. A user
    and user-proxy may be encoded in the token and should not be also
    given
user: str or None
    If given, assert the user name to connect with
password: str or None
    If given, assert the password to use for basic auth. If password
    is provided, user must be provided also
proxy_to: str or None
    If given, the user has the authority to proxy, and this value is
    the user in who's name actions are taken
kerb_kwargs: dict
    Any extra arguments for HTTPKerberosAuth, see
    `<https://github.com/requests/requests-kerberos/blob/master/requests_kerberos/kerberos_.py>`_
data_proxy: dict, callable or None
    If given, map data-node addresses. This can be necessary if the
    HDFS cluster is behind a proxy, running on Docker or otherwise has
    a mismatch between the host-names given by the name-node and the
    address by which to refer to them from the client. If a dict,
    maps host names ``host->data_proxy[host]``; if a callable, full
    URLs are passed, and function must conform to
    ``url->data_proxy(url)``.
use_https: bool
    Whether to connect to the Name-node using HTTPS instead of HTTP
session_cert: str or Tuple[str, str] or None
    Path to a certificate file, or tuple of (cert, key) files to use
    for the requests.Session
session_verify: str, bool or None
    Path to a certificate file to use for verifying the requests.Session.
kwargs
Nhttpshttpz://:z/webhdfs/v1z_If passing a delegation token, must not set user or proxy_to, as these are encoded in the token
delegationzQIf passing a password, the user must also beset in order to set up the basic-authz	user.namedoaszJIf using Kerberos auth, do not specify the user, this is handled by kinit.webhdfs_ )_cachedsuper__init__urlkerbkerb_kwargsparsproxy
ValueErroruserpasswordsession_certsession_verify_connectr	   _fsid)selfhostportkerberostokenr   r   proxy_tor   
data_proxy	use_httpsr    r!   kwargs	__class__s   &&&&&&&&&&&&&,v/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/fsspec/implementations/webhdfs.pyr   WebHDFS.__init__.   s4   r <<<"6"!*g7s4&${S	&,,"	%%2
8#7  
 ',IIl#	 | <   )-		+& (IIf(2 
 ), 456
    c                    V P                   # N)r#   r$   s   &r.   fsidWebHDFS.fsid   s    zzr0   c                   \         P                  ! 4       V n        V P                  '       d   V P                  V P                  n        V P
                  V P                  n        V P                  '       d(   ^ RIH	p V! R/ V P                  B V P                  n        V P                  eE   V P                  e5   ^ RIHp V! V P                  V P                  4      V P                  n        R# R# R# )r   )HTTPKerberosAuthN)HTTPBasicAuthr   )requestsSessionsessionr    certr!   verifyr   requests_kerberosr7   r   authr   r   requests.authr8   )r$   r7   r8   s   &  r.   r"   WebHDFS._connect   s    '') $ 1 1DLL"11999: 0 D43C3C DDLL99 T]]%>3 -dii GDLL &? r0   c                   Ve   V P                  V4      MRpV P                  V P                  \        VRR7      ,           4      pVP	                  4       pVP                  V P                  4       VP                  4       VR&   \        P                  RWr4       V P                  P                  VP                  4       VVVVR7      p	V	P                  R
9   ds    V	P                  4       p
V
R,          R,          pV
R,          R	,          pVR9   d   \        V4      hVR9   d   \        V4      hVR9   d   \!        V4      h\#        V4      hV	P'                  4        V	#   \        \$        3 d     L%i ; i)N z/=)safeopzsending %s with %s)methodr   paramsdataallow_redirectsRemoteExceptionmessage	exception)i  i  i  i  i  )IllegalArgumentExceptionUnsupportedOperationException)SecurityExceptionAccessControlException)FileNotFoundException)_strip_protocol_apply_proxyr   r   copyupdater   upperloggerdebugr;   requeststatus_codejsonr   PermissionErrorFileNotFoundErrorRuntimeErrorKeyErrorraise_for_status)r$   rE   rF   pathrH   redirectr,   r   argsouterrmsgexps   &&&&&&,      r.   _callWebHDFS._call   sQ   -1-=t##D)25D+A AB{{}DIIXXZT
)37ll""<<>$ # 
 ??77,hhj+,Y7+,[9 WW$S/)KK)#..55+C00&s++
 ) s   0E E0/E0c                h    T;'       g    V P                   p\        V VVVV P                  VVVR7      # )a  

Parameters
----------
path: str
    File location
mode: str
    'rb', 'wb', etc.
block_size: int
    Client buffer size for read-ahead or write buffer
autocommit: bool
    If False, writes to temporary file that only gets put in final
    location upon commit
replication: int
    Number of copies of file on the cluster, write mode only
permissions: str or int
    posix permissions, write mode only
kwargs

Returns
-------
WebHDFile instance
)mode
block_sizetempdir
autocommitreplicationpermissions)	blocksize	WebHDFilerm   )r$   ra   rk   rl   rn   ro   rp   r,   s   &&&&&&&,r.   _openWebHDFS._open   s?    B  114>>
!LL!##	
 		
r0   c                R    V R ,          P                  4       V R &   V R,          V R&   V # )typelengthsize)lower)infos   &r.   _process_infoWebHDFS._process_info   s*    F|))+VH~Vr0   c                &    \        V4      R ,          # ra   )r   )clsra   s   &&r.   rR   WebHDFS._strip_protocol   s    $T*622r0   c                    \        V 4      pVP                  R R4       VP                  RR4       RV9   d   VP                  R4      VR&   V# )ra   Nprotocolusernamer   )r   pop)urlpathrd   s   & r.   _get_kwargs_from_urlsWebHDFS._get_kwargs_from_urls  sG    #G,
D!''*-CK
r0   c                    V P                  R VR7      pVP                  4       R,          pWR&   V P                  V4      # )GETFILESTATUSr~   
FileStatusname)rh   r[   r{   )r$   ra   rd   rz   s   &&  r.   rz   WebHDFS.info
  s<    jjtj4xxz,'V!!$''r0   c                    V P                  V4      pVP                  RR4      pVe   \        P                  ! VR,          4      # \	        R4      h)z=Return the created timestamp of a file as a datetime.datetimemodificationTimeN  z5Could not retrieve creation time (modification time).rz   getr   fromtimestampr^   r$   ra   rz   mtimes   &&  r.   createdWebHDFS.created  sI     yy+T2))%$,77RSSr0   c                    V P                  V4      pVP                  RR4      pVe   \        P                  ! VR,          4      # \	        R4      h)z>Return the modified timestamp of a file as a datetime.datetimer   Nr   z%Could not retrieve modification time.r   r   s   &&  r.   modifiedWebHDFS.modified  sG    yy+T2))%$,77BCCr0   c                :   V P                  R VR7      pVP                  4       R,          R,          pV F=  pV P                  V4       VP                  R4      R,           VR,          ,           VR&   K?  	  V'       d   \	        VR R7      # \	        R	 V 4       4      # )

LISTSTATUSr~   FileStatusesr   /
pathSuffixr   c                     V R ,          # )r   r   )is   &r.   <lambda>WebHDFS.ls.<locals>.<lambda>(  s    qyr0   )keyc              3   2   "   T F  qR ,          x  K  	  R# 5i)r   Nr   ).0rz   s   & r.   	<genexpr>WebHDFS.ls.<locals>.<genexpr>*  s     954v,,5s   )rh   r[   r{   rstripsorted)r$   ra   detailr,   rd   infosrz   s   &&&,   r.   ls
WebHDFS.ls!  s    jjDj1
>*<8Dt$;;s+c1D4FFDL  %%89995999r0   c                V    V P                  RVR7      pVP                  4       R,          # )z8Total numbers of files, directories and bytes under pathGETCONTENTSUMMARYr~   ContentSummaryrh   r[   )r$   ra   rd   s   && r.   content_summaryWebHDFS.content_summary,  s'    jj,4j8xxz*++r0   c                b   V P                  RVRR7      pRVP                  9   de   V P                  VP                  R,          4      pV P                  P	                  V4      pVP                  4        VP                  4       R,          # VP                  4        VP                  4       R,          # )z/Checksum info of file, giving method and resultGETFILECHECKSUMF)ra   rb   LocationFileChecksum)rh   headersrS   r;   r   r`   r[   )r$   ra   rd   locationout2s   &&   r.   ukeyWebHDFS.ukey1  s    jj*jF$((Z)@AH<<##H-D!!#99;~..  "88:n--r0   c                R    V P                  R4      pVP                  4       R,          # )zGet user's home directoryGETHOMEDIRECTORYPathr   )r$   rd   s   & r.   home_directoryWebHDFS.home_directory=  s"    jj+,xxz&!!r0   c                    V'       d   V P                  RVR7      pMV P                  R4      pVP                  4       R,          pVf   \        R4      hVR,          # )zRetrieve token which can give the same authority to other uses

Parameters
----------
renewer: str or None
    User who may use this token; if None, will be current user
GETDELEGATIONTOKEN)renewerTokenz1No token available for this user/security context	urlString)rh   r[   r   )r$   r   rd   ts   &&  r.   get_delegation_tokenWebHDFS.get_delegation_tokenB  sT     **17*CC**12CHHJw9PQQ~r0   c                X    V P                  RRVR7      pVP                  4       R,          # )z/Make token live longer. Returns new expiry timeRENEWDELEGATIONTOKENputrF   r(   longr   )r$   r(   rd   s   && r.   renew_delegation_tokenWebHDFS.renew_delegation_tokenS  s(    jj/UjKxxz&!!r0   c                .    V P                  RRVR7       R# )z Stop the token from being usefulCANCELDELEGATIONTOKENr   r   Nrh   )r$   r(   s   &&r.   cancel_delegation_tokenWebHDFS.cancel_delegation_tokenX  s    

*5
Fr0   c                .    V P                  RRWR7       R# )zSet the permission at path

Parameters
----------
path: str
    location to set (file or directory)
mod: str or int
    posix epresentation or permission, give as oct string, e.g, '777'
    or 0o777
SETPERMISSIONr   )rF   ra   
permissionNr   )r$   ra   mods   &&&r.   chmodWebHDFS.chmod\  s     	

?5t
Lr0   c                V    / pVe   W$R&   Ve   W4R&   V P                   ! RRRRV/VB  R# )zChange owning user and/or groupNownergrouprF   r   ra   )SETOWNERr   )r$   ra   r   r   r,   s   &&&& r.   chownWebHDFS.chowni  s:    #7O#7O

AeA$A&Ar0   c                0    V P                  RVRVR7       R# )z
Set file replication factor

Parameters
----------
path: str
    File location (not for directories)
replication: int
    Number of copies of file on the cluster. Should be smaller than
    number of data nodes; normally 3 on most systems.
SETREPLICATIONr   )ra   rF   ro   Nr   )r$   ra   ro   s   &&&r.   set_replicationWebHDFS.set_replicationr  s     	

#$u+
Vr0   c                .    V P                  R RVR7       R# )MKDIRSr   )rF   ra   Nr   r$   ra   r,   s   &&,r.   mkdirWebHDFS.mkdir  s    

8E
5r0   c                x    VR J d#   V P                  V4      '       d   \        V4      hV P                  V4       R# )FN)existsFileExistsErrorr   )r$   ra   exist_oks   &&&r.   makedirsWebHDFS.makedirs  s/    uT!2!2!$''

4r0   c                .    V P                  R RWR7       R# )RENAMEr   )rF   ra   destinationNr   )r$   path1path2r,   s   &&&,r.   mv
WebHDFS.mv  s    

8E
Ir0   c                D    T P                  R RTV'       d   RMRR7       R# )DELETEdeletetruefalse)rF   ra   	recursiveNr   )r$   ra   r   r,   s   &&&,r.   rm
WebHDFS.rm  s#    

 )fw	 	 	
r0   c                (    V P                  V4       R # r2   )r   r   s   &&,r.   rm_fileWebHDFS.rm_file  s    r0   c           
     J   V P                  V4      ;_uu_ 4       pR P                  V P                  V4      R\        P                  ! ^4       2.4      p V P                  VR4      ;_uu_ 4       p\
        P                  ! WF4       RRR4       V P                  WR4        RRR4       R#   + '       g   i     L,; i  \         dG    \        \        4      ;_uu_ 4        T P                  T4       RRR4       h   + '       g   i     h ; ii ; i  + '       g   i     R# ; i)r   z.tmp.wbN)openjoin_parentsecrets	token_hexshutilcopyfileobjr   BaseExceptionr   r]   r   )r$   lpathrpathr,   lstream	tmp_fnamerstreams   &&&,   r.   cp_fileWebHDFS.cp_file  s    YYu$,,u"5w?P?PQS?T>U7V!WXIYYy$//7&&w8 0	) 
 0/ ! /00GGI& 1 10 sM   :DB=/B*	B=*B:5B=="DC:1	D:D
	DDD"	c                   V P                   '       d/   \        V P                   4      '       d   V P                  V4      pV# V P                   '       d6   V P                   P                  4        F  w  r#VP                  W#^4      pK  	  V# )   )r   callableitemsreplace)r$   r   kvs   &&  r.   rS   WebHDFS._apply_proxy  si    :::(4::..zz(+H
 	 ZZZ

((*#++A!4 +r0   )r#   r   r   r   r   r   r;   r    r!   r   r   )r
   webHDFS)i  FNNNNNNFNT)r   NNT)rbNTNNFr2   )NN),__name__
__module____qualname____firstlineno____doc__strtempfile
gettempdirrm   r   r   propertyr4   r"   rh   rs   staticmethodr{   classmethodrR   r   rz   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  rS   __static_attributes____classdictcell____classcell__r-   __classdict__s   @@r.   r   r      s     * (%%'(G#Hc7J  H$B+
Z  
 3 3  (TD	:,

."
""
GMBW6
J
 r0   r   c                   X   a a ] tR tRt oRtV 3R ltR
R ltR tR tR t	R t
R	tVtV ;t# )rr   i  z"A file living in HDFS over webHDFSc                  < \         SV `  ! W3/ VB  VP                  4       pVP                  R R4      f   VP	                  R R4       VP                  RR4      f   VP	                  RR4       VP	                  R R4      V n        VP	                  R4      pVP	                  RR4      RJ dU   V P                  V n        \        P                  P                  V\        \        P                  ! 4       4      4      V n        R# R# )rp   Nro   i  rm   rn   F)r   r   rT   r   r   rp   ra   targetosr  r$  uuiduuid4)r$   fsra   r,   rm   r-   s   &&&, r.   r   WebHDFile.__init__  s    ,V,::mT*2JJ}d+::mT*2JJ}d+!::mS9**Y'::lE*e3))DKWc$**,.?@DI 4r0   c                    V P                   P                  P                  V P                  V P                  P                  4       RR/R7      pVP                  4        R# )zWrite one part of a multi-block file upload

Parameters
==========
final: bool
    This is the last block, so should complete file, if
    self.autocommit is True.
content-typeapplication/octet-stream)rH   r   T)r5  r;   postr   buffergetvaluer`   )r$   finalrd   s   && r.   _upload_chunkWebHDFile._upload_chunk  sR     ggoo""MM%%'#%?@ # 

 	r0   c                p   V P                   P                  4       pRV P                  9   d   RRr2MRRr2RVR&   V P                  P                  ! W#V P
                  3RR	/VB pV P                  P                  VP                  R
,          4      pRV P                  9   d   V P                  P                  P                  VRR/R7      pVP                  4        V P                  P                  ! RRV P
                  3RR	/VB pV P                  P                  VP                  R
,          4      V n        R# R# )zCreate remote file/uploadaAPPENDPOSTCREATEPUTr   	overwriterb   Fr   wr8  r9  )r   N)r,   rT   rk   r5  rh   ra   rS   r   r;   r   r`   r   )r$   r,   rE   rF   rd   r   r   s   &      r.   _initiate_uploadWebHDFile._initiate_upload  s   !!#$))!6!5"(F;ggmmB		LELVL77''J(?@$))77??&&>3M"N ' D !!#77==6499WuWPVWD GG00j1IJDM r0   c                   \        V^ 4      p\        V P                  V4      pW8  g   WP                  8  d   R# V P                  P	                  RV P
                  WV,
          RR7      pVP                  4        RVP                  9   d^   VP                  R,          pV P                  P                  P                  V P                  P                  V4      4      pVP                  # VP                  # )r   r0   OPENF)ra   offsetrw   rb   r   )maxminrx   r5  rh   ra   r`   r   r;   r   rS   content)r$   startendrd   r   r   s   &&&   r.   _fetch_rangeWebHDFile._fetch_range  s    E1$))S!<5II-ggmm5uu  
 	${{:.H77??&&tww';';H'EFD<<;;r0   c                f    V P                   P                  V P                  V P                  4       R # r2   )r5  r   ra   r1  r3   s   &r.   commitWebHDFile.commit  s    

499dkk*r0   c                P    V P                   P                  V P                  4       R # r2   )r5  r   ra   r3   s   &r.   discardWebHDFile.discard  s    

499r0   )r   ra   rp   r1  r  )r  r   r!  r"  r#  r   r>  rH  rR  rU  rX  r*  r+  r,  r-  s   @@r.   rr   rr     s-     ,A"K( + r0   rr   )loggingr2  r  r
  r%  r3  
contextlibr   r   urllib.parser   r9   specr   r   utilsr   r	   	getLoggerrW   r   rr   r   r0   r.   <module>r`     sZ     	         ; 3			9	%V  VrI$ Ir0   