From 3c4865a6c84b3b0d3c4ad635ed500259e4475d83 Mon Sep 17 00:00:00 2001 From: cccs-eric Date: Mon, 12 Dec 2022 19:15:33 -0500 Subject: [PATCH 1/2] Fix missing resource path when FileNotFoundErrors are raised (#378) --- adlfs/spec.py | 31 ++++++++++++++++--------------- adlfs/tests/test_spec.py | 21 ++++++++++++++------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index db9d9b9a..0025fc20 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -7,6 +7,7 @@ import io import logging import os +import errno import re import warnings import weakref @@ -549,7 +550,7 @@ async def _info(self, path, refresh=False, **kwargs): if out: return out[0] else: - raise FileNotFoundError + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) def glob(self, path, **kwargs): return sync(self.loop, self._glob, path) @@ -711,8 +712,8 @@ async def _ls_blobs( outblobs.append(blob_) else: pass - except ResourceNotFoundError: - raise FileNotFoundError + except ResourceNotFoundError as e: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target_path) from e finalblobs = await self._details( outblobs, target_path=target_path, @@ -726,7 +727,7 @@ async def _ls_blobs( ) if not finalblobs: if not await self._exists(target_path): - raise FileNotFoundError + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target_path) return [] if not self.version_aware or finalblobs[0].get("is_current_version"): self.dircache[target_path] = finalblobs @@ -1372,25 +1373,25 @@ async def _pipe_file(self, path, value, overwrite=True, **kwargs): pipe_file = sync_wrapper(_pipe_file) async def _cat_file(self, path, start=None, end=None, **kwargs): - path = self._strip_protocol(path) + no_protocol_path = self._strip_protocol(path) if end is not None: start = start or 0 # download_blob requires start if length is provided. length = end - start else: length = None - container_name, path, version_id = self.split_path(path) + container_name, no_protocol_path, version_id = self.split_path(no_protocol_path) async with self.service_client.get_blob_client( - container=container_name, blob=path + container=container_name, blob=no_protocol_path ) as bc: try: stream = await bc.download_blob( offset=start, length=length, version_id=version_id ) except ResourceNotFoundError as e: - raise FileNotFoundError from e + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) from e except HttpResponseError as e: if version_id is not None: - raise FileNotFoundError from e + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) from e raise result = await stream.readall() return result @@ -1517,7 +1518,7 @@ async def _expand_path( out.add(fullpath) if not out: - raise FileNotFoundError + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) return list(sorted(out)) async def _put_file( @@ -1553,9 +1554,9 @@ async def _put_file( self.invalidate_cache() except ResourceExistsError: raise FileExistsError("File already exists!") - except ResourceNotFoundError: + except ResourceNotFoundError as e: if not await self._exists(container_name): - raise FileNotFoundError("Container does not exist.") + raise FileNotFoundError(errno.ENOENT, "Container does not exist.", container_name) from e await self._put_file(lpath, rpath, delimiter, overwrite) self.invalidate_cache() @@ -1581,7 +1582,7 @@ async def _cp_file(self, path1, path2, **kwargs): try: await blobclient2.start_copy_from_url(url) except ResourceNotFoundError as e: - raise FileNotFoundError from e + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path1) from e self.invalidate_cache(container1) self.invalidate_cache(container2) @@ -1615,7 +1616,7 @@ async def _get_file( ) await stream.readinto(my_blob) except ResourceNotFoundError as exception: - raise FileNotFoundError from exception + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rpath) from exception get_file = sync_wrapper(_get_file) @@ -1630,7 +1631,7 @@ async def _setxattrs(self, rpath, **kwargs): await bc.set_blob_metadata(metadata=kwargs) self.invalidate_cache(self._parent(rpath)) except Exception as e: - raise FileNotFoundError(f"File not found for {e}") + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rpath) from e setxattrs = sync_wrapper(_setxattrs) diff --git a/adlfs/tests/test_spec.py b/adlfs/tests/test_spec.py index 4cbbb5fb..39091d49 100644 --- a/adlfs/tests/test_spec.py +++ b/adlfs/tests/test_spec.py @@ -247,14 +247,17 @@ def test_ls(storage): ) # if not direct match is found throws error - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: fs.ls("not-a-container") + assert "not-a-container" in str(e.value) - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: fs.ls("data/not-a-directory/") + assert "data/not-a-directory" in str(e.value) - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: fs.ls("data/root/not-a-file.txt") + assert "data/root/not-a-file.txt" in str(e.value) def test_ls_no_listings_cache(storage): @@ -1400,11 +1403,13 @@ def test_cat_file_missing(storage): account_name=storage.account_name, connection_string=CONN_STR ) fs.mkdir("catdir") - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: fs.cat_file("catdir/not/exist") + assert "catdir/not/exist" in str(e.value) - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: fs.cat_file("does/not/exist") + assert "does/not/exist" in str(e.value) async def test_cat_file_versioned(storage, mocker): @@ -1426,8 +1431,9 @@ async def test_cat_file_versioned(storage, mocker): download_blob.reset_mock() download_blob.side_effect = HttpResponseError - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: await fs._cat_file("data/root/a/file.txt?versionid=invalid_version") + assert "data/root/a/file.txt?versionid=invalid_version" in str(e.value) @pytest.mark.skip( @@ -1685,5 +1691,6 @@ async def test_get_file_versioned(storage, mocker): download_blob.reset_mock() download_blob.side_effect = ResourceNotFoundError - with pytest.raises(FileNotFoundError): + with pytest.raises(FileNotFoundError) as e: await fs._get_file("data/root/a/file.txt?versionid=invalid_version", "file.txt") + assert "data/root/a/file.txt?versionid=invalid_version" in str(e.value) From fb80c1361c89102e670936430f5777d08f9e6f63 Mon Sep 17 00:00:00 2001 From: cccs-eric Date: Mon, 12 Dec 2022 19:25:14 -0500 Subject: [PATCH 2/2] Fix missing resource path when FileNotFoundErrors are raised (#378) --- adlfs/spec.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/adlfs/spec.py b/adlfs/spec.py index 0025fc20..ac5586da 100644 --- a/adlfs/spec.py +++ b/adlfs/spec.py @@ -4,10 +4,10 @@ from __future__ import absolute_import, division, print_function import asyncio +import errno import io import logging import os -import errno import re import warnings import weakref @@ -713,7 +713,9 @@ async def _ls_blobs( else: pass except ResourceNotFoundError as e: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target_path) from e + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), target_path + ) from e finalblobs = await self._details( outblobs, target_path=target_path, @@ -727,7 +729,9 @@ async def _ls_blobs( ) if not finalblobs: if not await self._exists(target_path): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), target_path) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), target_path + ) return [] if not self.version_aware or finalblobs[0].get("is_current_version"): self.dircache[target_path] = finalblobs @@ -1388,10 +1392,14 @@ async def _cat_file(self, path, start=None, end=None, **kwargs): offset=start, length=length, version_id=version_id ) except ResourceNotFoundError as e: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) from e + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path + ) from e except HttpResponseError as e: if version_id is not None: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) from e + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path + ) from e raise result = await stream.readall() return result @@ -1556,7 +1564,9 @@ async def _put_file( raise FileExistsError("File already exists!") except ResourceNotFoundError as e: if not await self._exists(container_name): - raise FileNotFoundError(errno.ENOENT, "Container does not exist.", container_name) from e + raise FileNotFoundError( + errno.ENOENT, "Container does not exist.", container_name + ) from e await self._put_file(lpath, rpath, delimiter, overwrite) self.invalidate_cache() @@ -1582,7 +1592,9 @@ async def _cp_file(self, path1, path2, **kwargs): try: await blobclient2.start_copy_from_url(url) except ResourceNotFoundError as e: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path1) from e + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path1 + ) from e self.invalidate_cache(container1) self.invalidate_cache(container2) @@ -1616,7 +1628,9 @@ async def _get_file( ) await stream.readinto(my_blob) except ResourceNotFoundError as exception: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rpath) from exception + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), rpath + ) from exception get_file = sync_wrapper(_get_file) @@ -1631,7 +1645,9 @@ async def _setxattrs(self, rpath, **kwargs): await bc.set_blob_metadata(metadata=kwargs) self.invalidate_cache(self._parent(rpath)) except Exception as e: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rpath) from e + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), rpath + ) from e setxattrs = sync_wrapper(_setxattrs)