diff --git a/tests/core/fixtures.py b/tests/core/fixtures.py new file mode 100644 index 000000000..a755cb0e5 --- /dev/null +++ b/tests/core/fixtures.py @@ -0,0 +1,31 @@ +import pytest + +from tests import utils +from waterbutler.core.path import WaterButlerPath + + +@pytest.fixture +def provider1(): + return utils.MockProvider1({'user': 'name'}, {'pass': 'word'}, {}) + + +@pytest.fixture +def provider2(): + return utils.MockProvider2({'user': 'name'}, {'pass': 'phrase'}, {}) + + +@pytest.fixture +def dest_path(): + return WaterButlerPath('/older folder/new folder/') + +@pytest.fixture +def folder_children(): + return [utils.MockFileMetadata(), utils.MockFolderMetadata(), utils.MockFileMetadata()] + +@pytest.fixture +def src_path(): + return WaterButlerPath('/folder with children/') + +@pytest.fixture +def mock_folder(): + return utils.MockFolderMetadata() diff --git a/tests/core/test_provider.py b/tests/core/test_provider.py index 123888ca0..115d112b4 100644 --- a/tests/core/test_provider.py +++ b/tests/core/test_provider.py @@ -5,16 +5,14 @@ from waterbutler.core import metadata from waterbutler.core import exceptions - -@pytest.fixture -def provider1(): - return utils.MockProvider1({'user': 'name'}, {'pass': 'word'}, {}) - - -@pytest.fixture -def provider2(): - return utils.MockProvider2({'user': 'name'}, {'pass': 'phrase'}, {}) - +from tests.core.fixtures import ( + provider1, + provider2, + src_path, + dest_path, + folder_children, + mock_folder +) class TestBaseProvider: @@ -481,3 +479,76 @@ def test_build_range_header(self, provider1): assert 'bytes=10-' == provider1._build_range_header((10, None)) assert 'bytes=10-100' == provider1._build_range_header((10, 100)) assert 'bytes=-255' == provider1._build_range_header((None, 255)) + + +class TestFolderFileOp: + + @pytest.mark.asyncio + async def test_folder_file_op_overwrite(self, + provider1, + provider2, + src_path, + dest_path, + mock_folder, + folder_children): + provider2.create_folder = utils.MockCoroutine(return_value=mock_folder) + provider1.metadata = utils.MockCoroutine(return_value=folder_children) + provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True)) + + folder, created = await provider1._folder_file_op(provider1.copy, + provider2, + src_path, + dest_path) + + provider2.create_folder.assert_called_with(dest_path, folder_precheck=False) + provider1.metadata.assert_called_with(src_path) + + assert (folder, created) == (mock_folder, False) + + # The order of these lists does not matter, but they should contain all the same elements. + assert all(child for child in folder.children if child in folder_children) + assert all(child for child in folder_children if child in folder.children) + + @pytest.mark.asyncio + async def test_folder_file_op_new(self, + provider1, + provider2, + src_path, + dest_path, + mock_folder, + folder_children): + provider2.create_folder = utils.MockCoroutine(return_value=mock_folder) + provider2.delete = utils.MockCoroutine(side_effect=exceptions.ProviderError('test message', + code=404)) + provider1.metadata = utils.MockCoroutine(return_value=folder_children) + provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True)) + + folder, created = await provider1._folder_file_op(provider1.copy, + provider2, + src_path, + dest_path) + provider2.create_folder.assert_called_with(dest_path, folder_precheck=False) + provider1.metadata.assert_called_with(src_path) + + assert (folder, created) == (mock_folder, True) + + # The order of these lists does not matter, but they should contain all the same elements. + assert all(child for child in folder.children if child in folder_children) + assert all(child for child in folder_children if child in folder.children) + + @pytest.mark.asyncio + async def test_folder_file_op_raises(self, + provider1, + provider2, + src_path, + dest_path, + mock_folder, + folder_children): + provider2.create_folder = utils.MockCoroutine(return_value=mock_folder) + provider2.delete = utils.MockCoroutine(side_effect=exceptions.ProviderError('test message', + code=500)) + provider1.metadata = utils.MockCoroutine(return_value=folder_children) + provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True)) + + with pytest.raises(exceptions.ProviderError): + await provider1._folder_file_op(provider1.copy, provider2, src_path, dest_path) diff --git a/tests/providers/bitbucket/test_metadata.py b/tests/providers/bitbucket/test_metadata.py index 6de003423..076431ef5 100644 --- a/tests/providers/bitbucket/test_metadata.py +++ b/tests/providers/bitbucket/test_metadata.py @@ -27,6 +27,7 @@ def test_build_file_metadata(self, file_metadata, owner, repo): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == full_path assert metadata.name == name assert metadata.path == full_path assert metadata.kind == 'file' diff --git a/tests/providers/box/test_metadata.py b/tests/providers/box/test_metadata.py index b01d2ad5f..a28d622d6 100644 --- a/tests/providers/box/test_metadata.py +++ b/tests/providers/box/test_metadata.py @@ -16,6 +16,7 @@ def test_file_metadata(self, root_provider_fixtures): item = root_provider_fixtures['file_metadata']['entries'][0] dest_path = WaterButlerPath('/charmander/name.txt', _ids=('0', item['id'], item['id'])) data = BoxFileMetadata(item, dest_path) + assert data.id == '5000948880' assert data.name == 'tigers.jpeg' assert data.path == '/5000948880' assert data.provider == 'box' diff --git a/tests/providers/cloudfiles/test_metadata.py b/tests/providers/cloudfiles/test_metadata.py index b69e44cc3..127d9c00f 100644 --- a/tests/providers/cloudfiles/test_metadata.py +++ b/tests/providers/cloudfiles/test_metadata.py @@ -39,6 +39,7 @@ def test_header_metadata(self, file_header_metadata_txt): path = WaterButlerPath('/file.txt') data = CloudFilesHeaderMetadata(file_header_metadata_txt, path.path) + assert data.id == '/file.txt' assert data.name == 'file.txt' assert data.path == '/file.txt' assert data.provider == 'cloudfiles' diff --git a/tests/providers/dataverse/test_metadata.py b/tests/providers/dataverse/test_metadata.py index ccb139087..8da1ba975 100644 --- a/tests/providers/dataverse/test_metadata.py +++ b/tests/providers/dataverse/test_metadata.py @@ -32,6 +32,7 @@ def test_revision_metadata(self, revision_metadata_object): class TestFileMetadata: def test_file_metadata(self, file_metadata_object): + assert file_metadata_object.id == '20' assert file_metadata_object.is_file assert not file_metadata_object.is_folder assert file_metadata_object.provider == 'dataverse' diff --git a/tests/providers/dropbox/test_metadata.py b/tests/providers/dropbox/test_metadata.py index 55684d28a..7700bea3b 100644 --- a/tests/providers/dropbox/test_metadata.py +++ b/tests/providers/dropbox/test_metadata.py @@ -12,6 +12,7 @@ class TestDropboxMetadata: def test_file_metadata(self, provider_fixtures): data = DropboxFileMetadata(provider_fixtures['file_metadata'], '/Photos') + assert data.id == 'id:8y8sAJlrhuAAAAAAAAAAAQ' assert data.name == 'Getting_Started.pdf' assert data.path == '/Getting_Started.pdf' assert data.size == 124778 @@ -88,6 +89,7 @@ def test_file_metadata(self, provider_fixtures): def test_folder_metadata(self, provider_fixtures): data = DropboxFolderMetadata(provider_fixtures['folder_metadata'], '/Photos') + assert data.id == 'id:67BLXqRKo-gAAAAAAAADZg' assert data.name == 'newfolder' assert data.path == '/newfolder/' assert data.etag is None diff --git a/tests/providers/filesystem/test_metadata.py b/tests/providers/filesystem/test_metadata.py index 7524e0ffc..b3ff9a815 100644 --- a/tests/providers/filesystem/test_metadata.py +++ b/tests/providers/filesystem/test_metadata.py @@ -39,6 +39,7 @@ class TestMetadata: def test_file_metadata(self, file_metadata): data = FileSystemFileMetadata(file_metadata, '/') + assert data.id == '/code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9' assert data.path == '/code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9' assert data.provider == 'filesystem' assert data.modified == 'Wed, 20 Sep 2017 15:16:02 +0000' diff --git a/tests/providers/github/test_metadata.py b/tests/providers/github/test_metadata.py index cc89d7e49..9854743af 100644 --- a/tests/providers/github/test_metadata.py +++ b/tests/providers/github/test_metadata.py @@ -20,6 +20,7 @@ def test_build_file_metadata_from_tree(self, metadata_fixtures): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == '/README.md' assert metadata.name == 'README.md' assert metadata.path == '/README.md' assert metadata.modified is None @@ -45,6 +46,7 @@ def test_build_file_metadata_from_contents(self, metadata_fixtures): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == '/epsilon' assert metadata.name == 'epsilon' assert metadata.path == '/epsilon' assert metadata.modified is None @@ -70,6 +72,7 @@ def test_build_folder_metadata_from_tree(self, metadata_fixtures): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == '/foldera/folderb/lorch/' assert metadata.name == 'lorch' assert metadata.path == '/foldera/folderb/lorch/' assert metadata.extra == {} @@ -84,6 +87,7 @@ def test_build_folder_metadata_from_content(self, metadata_fixtures): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == '/manyfiles/' assert metadata.name == 'manyfiles' assert metadata.path == '/manyfiles/' assert metadata.extra == {} @@ -98,6 +102,7 @@ def test_file_metadata_with_ref(self, metadata_fixtures): except Exception as exc: pytest.fail(str(exc)) + assert metadata.id == '/README.md' assert metadata.name == 'README.md' assert metadata.path == '/README.md' assert metadata.modified is None diff --git a/tests/providers/onedrive/test_metadata.py b/tests/providers/onedrive/test_metadata.py index c2b599c63..fa2ab66ef 100644 --- a/tests/providers/onedrive/test_metadata.py +++ b/tests/providers/onedrive/test_metadata.py @@ -23,6 +23,7 @@ def test_build_file_metadata(self, root_provider_fixtures): 'etag': 'aRjRENTBFNDAwREZFN0Q0RSEyOTEuMg', 'webView': 'https://1drv.ms/t/s!AE59_g1ADtX0giM', } + assert metadata.id == 'F4D50E400DFE7D4E!291' assert metadata.name == 'toes.txt' assert metadata.path == '/{}'.format(root_provider_fixtures['file_id']) assert metadata.size == 11 @@ -45,6 +46,7 @@ def test_build_folder_metadata(self, root_provider_fixtures): metadata = OneDriveFolderMetadata(root_provider_fixtures['folder_metadata'], od_path) assert metadata.provider == 'onedrive' + assert metadata.id == 'F4D50E400DFE7D4E!290' assert metadata.name == 'teeth' assert metadata.path == '/F4D50E400DFE7D4E!290/' assert metadata.etag == 'aRjRENTBFNDAwREZFN0Q0RSEyOTAuMA' diff --git a/tests/providers/osfstorage/test_metadata.py b/tests/providers/osfstorage/test_metadata.py index 5234ac490..40c092464 100644 --- a/tests/providers/osfstorage/test_metadata.py +++ b/tests/providers/osfstorage/test_metadata.py @@ -13,6 +13,7 @@ class TestFileMetadata: def test_file_metadata(self, file_metadata, file_metadata_object): assert file_metadata_object.provider == 'osfstorage' + assert file_metadata_object.id == '59a9b628b7d1c903ab5a8f52' assert file_metadata_object.name == 'doc.rst' assert file_metadata_object.path == '/59a9b628b7d1c903ab5a8f52' assert str(file_metadata_object.materialized_path) == '/doc.rst' @@ -69,6 +70,7 @@ class TestFolderMetadata: def test_folder_metadata(self, folder_metadata_object): assert folder_metadata_object.provider == 'osfstorage' + assert folder_metadata_object.id == '59c0054cb7d1c90114c456af' assert folder_metadata_object.name == 'New Folder' assert folder_metadata_object.path == '/59c0054cb7d1c90114c456af/' assert str(folder_metadata_object.materialized_path) == '/New Folder/' diff --git a/tests/providers/owncloud/test_metadata.py b/tests/providers/owncloud/test_metadata.py index 4d919f88d..098dca326 100644 --- a/tests/providers/owncloud/test_metadata.py +++ b/tests/providers/owncloud/test_metadata.py @@ -16,6 +16,7 @@ class TestFileMetadata: def test_file_metadata(self, file_metadata_object): assert file_metadata_object.provider == 'owncloud' + assert file_metadata_object.id == '/Documents/dissertation.aux' assert file_metadata_object.name == 'dissertation.aux' assert file_metadata_object.path == '/Documents/dissertation.aux' assert file_metadata_object.materialized_path == '/Documents/dissertation.aux' @@ -42,6 +43,7 @@ def test_file_metadata(self, file_metadata_object): def test_file_metadata_less_info(self, file_metadata_object_less_info): assert file_metadata_object_less_info.provider == 'owncloud' + assert file_metadata_object_less_info.id == '/Documents/dissertation.aux' assert file_metadata_object_less_info.name == 'dissertation.aux' assert file_metadata_object_less_info.path == '/Documents/dissertation.aux' assert file_metadata_object_less_info.materialized_path == '/Documents/dissertation.aux' @@ -70,6 +72,7 @@ class TestFolderMetadata: def test_folder_metadata(self, folder_metadata_object): assert folder_metadata_object.provider == 'owncloud' + assert folder_metadata_object.id == '/' assert folder_metadata_object.name == 'Documents' assert folder_metadata_object.path == '/' assert folder_metadata_object.materialized_path == '/' @@ -92,6 +95,7 @@ def test_folder_metadata(self, folder_metadata_object): def test_folder_metadata_less_info(self, folder_metadata_object_less_info): assert folder_metadata_object_less_info.provider == 'owncloud' + assert folder_metadata_object_less_info.id == '/' assert folder_metadata_object_less_info.name == 'Documents' assert folder_metadata_object_less_info.path == '/' assert folder_metadata_object_less_info.materialized_path == '/' diff --git a/tests/providers/s3/test_metadata.py b/tests/providers/s3/test_metadata.py index 2b2c68d92..23a76429d 100644 --- a/tests/providers/s3/test_metadata.py +++ b/tests/providers/s3/test_metadata.py @@ -15,6 +15,7 @@ class TestFileMetadataHeaders: def test_file_metadata_headers(self, file_metadata_headers_object, file_header_metadata): assert not file_metadata_headers_object.is_folder assert file_metadata_headers_object.is_file + assert file_metadata_headers_object.id == '/test-path' assert file_metadata_headers_object.name == 'test-path' assert file_metadata_headers_object.path == '/test-path' assert file_metadata_headers_object.materialized_path == '/test-path' @@ -46,6 +47,7 @@ def test_file_metadata(self, file_metadata_object): assert file_metadata_object.is_file assert file_metadata_object.kind == 'file' + assert file_metadata_object.id == '/my-image.jpg' assert file_metadata_object.name == 'my-image.jpg' assert file_metadata_object.path == '/my-image.jpg' assert file_metadata_object.materialized_path == '/my-image.jpg' @@ -81,6 +83,7 @@ def test_folder_metadata(self, folder_metadata_object): assert not folder_metadata_object.is_file assert folder_metadata_object.kind == 'folder' + assert folder_metadata_object.id == '/photos/' assert folder_metadata_object.name == 'photos' assert folder_metadata_object.path == '/photos/' assert folder_metadata_object.materialized_path == '/photos/' @@ -103,6 +106,7 @@ def test_folder_key_metadata(self, folder_key_metadata_object): assert not folder_key_metadata_object.is_file assert folder_key_metadata_object.kind == 'folder' + assert folder_key_metadata_object.id == '/naptime/' assert folder_key_metadata_object.name == 'naptime' assert folder_key_metadata_object.path == '/naptime/' diff --git a/tests/utils.py b/tests/utils.py index e91b45a98..e85be881a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,6 +26,7 @@ async def __call__(self, *args, **kwargs): class MockFileMetadata(metadata.BaseFileMetadata): + id = '/Foo.name' provider = 'MockProvider' name = 'Foo.name' size = 1337 @@ -41,6 +42,7 @@ def __init__(self): class MockFolderMetadata(metadata.BaseFolderMetadata): + id = '/Foo.name' provider = 'MockProvider' name = 'Bar' size = 1337 diff --git a/waterbutler/core/metadata.py b/waterbutler/core/metadata.py index 97f3c3856..2d721d449 100644 --- a/waterbutler/core/metadata.py +++ b/waterbutler/core/metadata.py @@ -133,6 +133,16 @@ def kind(self) -> str: """ `file` or `folder` """ raise NotImplementedError + @property + @abc.abstractmethod + def id(self) -> str: + """ This property consolidates path-based and id-based file/folder/object identity under a + single one: ``id``, which makes inter provider move/copy actions easier. For path-based + providers, the value is identical to ``path`` property. For id-based providers, it is the + raw identity by definition (``self.raw['id']`` for most providers). + """ + raise NotImplementedError + @property @abc.abstractmethod def name(self) -> str: diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index 84e44d8dd..143243067 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -335,7 +335,7 @@ async def _folder_file_op(self, folder = await dest_provider.create_folder(dest_path, folder_precheck=False) - dest_path = await dest_provider.revalidate_path(dest_path.parent, dest_path.name, folder=dest_path.is_dir) + folder_path = dest_provider.path_from_metadata(dest_path.parent, folder) folder.children = [] items = await self.metadata(src_path) # type: ignore @@ -349,9 +349,8 @@ async def _folder_file_op(self, futures.append(asyncio.ensure_future( func( dest_provider, - # TODO figure out a way to cut down on all the requests made here - (await self.revalidate_path(src_path, item.name, folder=item.is_folder)), - (await dest_provider.revalidate_path(dest_path, item.name, folder=item.is_folder)), + self.path_from_metadata(src_path, item), + folder_path.child(item.name, None, folder=item.is_folder), handle_naming=False, ) )) @@ -359,9 +358,6 @@ async def _folder_file_op(self, if item.is_folder: await futures[-1] - if not futures: - continue - done, _ = await asyncio.wait(futures, return_when=asyncio.FIRST_EXCEPTION) for fut in done: @@ -673,9 +669,8 @@ async def validate_path(self, path: str, **kwargs) -> wb_path.WaterButlerPath: def path_from_metadata(self, parent_path: wb_path.WaterButlerPath, - meta_data: wb_metadata.BaseMetadata) -> wb_path.WaterButlerPath: - return parent_path.child(meta_data.name, _id=meta_data.path.strip('/'), - folder=meta_data.is_folder) + metadata: wb_metadata.BaseMetadata) -> wb_path.WaterButlerPath: + return parent_path.child(metadata.name, _id=metadata.id, folder=metadata.is_folder) async def revisions(self, path: wb_path.WaterButlerPath, **kwargs): """Return a list of :class:`.BaseFileRevisionMetadata` objects representing the revisions diff --git a/waterbutler/providers/bitbucket/metadata.py b/waterbutler/providers/bitbucket/metadata.py index 77754825a..8b2960ef3 100644 --- a/waterbutler/providers/bitbucket/metadata.py +++ b/waterbutler/providers/bitbucket/metadata.py @@ -28,6 +28,11 @@ def __init__(self, raw, path_obj, owner=None, repo=None): def provider(self): return 'bitbucket' + # Bitbucket is a path-based provider and uses the default ``revalidate_path()``. + @property + def id(self): + return self.path + @property def path(self): return self.build_path() diff --git a/waterbutler/providers/box/metadata.py b/waterbutler/providers/box/metadata.py index bfcc1ff4a..f051ac95c 100644 --- a/waterbutler/providers/box/metadata.py +++ b/waterbutler/providers/box/metadata.py @@ -11,6 +11,11 @@ def __init__(self, raw, path_obj): def provider(self): return 'box' + # Box is an id-based provider and uses its own ``revalidate_path()``. + @property + def id(self): + return self.raw['id'] + @property def materialized_path(self): return str(self._path_obj) diff --git a/waterbutler/providers/cloudfiles/metadata.py b/waterbutler/providers/cloudfiles/metadata.py index aa13c1ed9..fd076ea0e 100644 --- a/waterbutler/providers/cloudfiles/metadata.py +++ b/waterbutler/providers/cloudfiles/metadata.py @@ -9,6 +9,14 @@ class BaseCloudFilesMetadata(metadata.BaseMetadata): def provider(self): return 'cloudfiles' + # CloudFiles is a path-based provider. However, it never called ``revalidate_path()`` even when + # it was the backend storage provider for OSFStorage. The provider is no longer active until it + # is upgraded to an addon provider for the users. + @property + def id(self): + # TODO: Do we need this ``id`` to be actually set? + return self.path + class CloudFilesFileMetadata(BaseCloudFilesMetadata, metadata.BaseFileMetadata): diff --git a/waterbutler/providers/dataverse/metadata.py b/waterbutler/providers/dataverse/metadata.py index 125325dbf..0fcf26203 100644 --- a/waterbutler/providers/dataverse/metadata.py +++ b/waterbutler/providers/dataverse/metadata.py @@ -7,6 +7,14 @@ class BaseDataverseMetadata(metadata.BaseMetadata): def provider(self): return 'dataverse' + # TODO: Should this be in ``DataverseFileMetadata``? + # TODO: Does ``self.raw['id']`` exists for ``DataverseDatasetMetadata``? + # TODO: DataVerse's ``revalidate_path()`` looks very different. Is this an issue? + # DataVerse is an id-based provider and has its own ``revalidate_path()``. + @property + def id(self): + return str(self.raw['id']) + class DataverseFileMetadata(BaseDataverseMetadata, metadata.BaseFileMetadata): diff --git a/waterbutler/providers/dropbox/metadata.py b/waterbutler/providers/dropbox/metadata.py index 42b1e618a..8f3ffda70 100644 --- a/waterbutler/providers/dropbox/metadata.py +++ b/waterbutler/providers/dropbox/metadata.py @@ -13,6 +13,12 @@ def __init__(self, raw, folder): def provider(self): return 'dropbox' + # TODO: Is Dropbox path-based instead of id-based? + # TODO: Dropbox does not have a ``revalidate_path()``, should we use path instead of id? + @property + def id(self): + return self.raw['id'] + def build_path(self, path): # TODO write a test for this if path.lower().startswith(self._folder.lower()): diff --git a/waterbutler/providers/figshare/metadata.py b/waterbutler/providers/figshare/metadata.py index 9fae70bf6..72378a84f 100644 --- a/waterbutler/providers/figshare/metadata.py +++ b/waterbutler/providers/figshare/metadata.py @@ -20,6 +20,7 @@ def __init__(self, raw, raw_file=None): else: self.raw_file = self.raw['files'][0] + # FigShare are id-based providers and use their own ``revalidate_path()``. @property def id(self): return self.raw_file['id'] @@ -116,6 +117,7 @@ class FigshareFolderMetadata(BaseFigshareMetadata, metadata.BaseFolderMetadata): considered folders. """ + # FigShare are id-based providers @property def id(self): return self.raw['id'] diff --git a/waterbutler/providers/filesystem/metadata.py b/waterbutler/providers/filesystem/metadata.py index 81ab0b129..3a54230d2 100644 --- a/waterbutler/providers/filesystem/metadata.py +++ b/waterbutler/providers/filesystem/metadata.py @@ -13,6 +13,11 @@ def __init__(self, raw, folder): def provider(self): return 'filesystem' + # FileSystem is a path-based provider and uses the default ``revalidate_path()``. + @property + def id(self): + return self.path + def build_path(self, path): # TODO write a test for this if path.lower().startswith(self._folder.lower()): diff --git a/waterbutler/providers/github/metadata.py b/waterbutler/providers/github/metadata.py index 9b20c9baa..2faf259fa 100644 --- a/waterbutler/providers/github/metadata.py +++ b/waterbutler/providers/github/metadata.py @@ -23,6 +23,12 @@ def __init__(self, raw, commit=None, ref=None): def provider(self): return 'github' + # GitHub is a path-based provider. However, it has its own ``revalidate_path()``. + # TODO: Investigate if this is an issue. + @property + def id(self): + return self.path + @property def extra(self): ret = {} diff --git a/waterbutler/providers/gitlab/metadata.py b/waterbutler/providers/gitlab/metadata.py index f5f6096f4..681527baa 100644 --- a/waterbutler/providers/gitlab/metadata.py +++ b/waterbutler/providers/gitlab/metadata.py @@ -19,6 +19,11 @@ def __init__(self, raw, path): def provider(self) -> str: return 'gitlab' + # GitLab is a path-based provider and uses the default ``revalidate_path()``. + @property + def id(self) -> str: + return self.path + @property def path(self) -> str: return self.build_path() diff --git a/waterbutler/providers/googlecloud/metadata.py b/waterbutler/providers/googlecloud/metadata.py index a6f24adf9..3011546b5 100644 --- a/waterbutler/providers/googlecloud/metadata.py +++ b/waterbutler/providers/googlecloud/metadata.py @@ -27,6 +27,13 @@ class BaseGoogleCloudMetadata(metadata.BaseMetadata, metaclass=abc.ABCMeta): def provider(self) -> str: return 'googlecloud' + # GoogleCloud is a path-based provider. However, it never calls ``revalidate_path()`` since it + # is the backend storage provider for OSFStorage. + @property + def id(self) -> str: + # TODO: Do we need this ``id`` to be actually set? + return self.path + @property def path(self) -> str: return self.build_path(self.raw.get('object_name', None)) diff --git a/waterbutler/providers/googledrive/metadata.py b/waterbutler/providers/googledrive/metadata.py index 5e4fbafa8..d162aeae3 100644 --- a/waterbutler/providers/googledrive/metadata.py +++ b/waterbutler/providers/googledrive/metadata.py @@ -32,6 +32,7 @@ def __init__(self, raw, path): super().__init__(raw, path) self._path._is_folder = True + # GoogleDrive is an id-based provider and uses its own ``revalidate_path()``. @property def id(self): return self.raw['id'] @@ -61,6 +62,7 @@ class GoogleDriveFileMetadata(BaseGoogleDriveMetadata, metadata.BaseFileMetadata [2] https://developers.google.com/drive/v2/reference/files/get """ + # GoogleDrive is an id-based provider and uses its own ``revalidate_path()``. @property def id(self): return self.raw['id'] diff --git a/waterbutler/providers/onedrive/metadata.py b/waterbutler/providers/onedrive/metadata.py index 3bd3efc9a..266271a2c 100644 --- a/waterbutler/providers/onedrive/metadata.py +++ b/waterbutler/providers/onedrive/metadata.py @@ -16,6 +16,11 @@ def __init__(self, raw, path_obj): def provider(self): return 'onedrive' + # OneDrive is an id-based provider and uses its own ``revalidate_path()``. + @property + def id(self): + return self.raw.get('id') + @property def materialized_path(self): return str(self._path_obj) diff --git a/waterbutler/providers/osfstorage/metadata.py b/waterbutler/providers/osfstorage/metadata.py index be5f00402..b21411077 100644 --- a/waterbutler/providers/osfstorage/metadata.py +++ b/waterbutler/providers/osfstorage/metadata.py @@ -16,6 +16,12 @@ def __init__(self, raw, materialized): super().__init__(raw) self._materialized = materialized + # TODO: Is OSFStorage also path-based? + # OSFStorage is an id-based provider and uses its own ``revalidate_path()``. + @property + def id(self): + return self.raw['id'] + @property def name(self): return self.raw['name'] diff --git a/waterbutler/providers/owncloud/metadata.py b/waterbutler/providers/owncloud/metadata.py index 8ca767ab6..03d7e2b86 100644 --- a/waterbutler/providers/owncloud/metadata.py +++ b/waterbutler/providers/owncloud/metadata.py @@ -13,14 +13,18 @@ def __init__(self, href, folder, attributes=None): def provider(self): return 'owncloud' + # OwnCloud is a path-based provider and uses the default ``revalidate_path()``. + @property + def id(self): + return self.path + @property def name(self): return self._href.strip('/').split('/')[-1] @property def path(self): - path = self._href[len(self._folder) - 1:] - return path + return self._href[len(self._folder) - 1:] @property def size(self): diff --git a/waterbutler/providers/s3/metadata.py b/waterbutler/providers/s3/metadata.py index bcdbdea4a..a4249f194 100644 --- a/waterbutler/providers/s3/metadata.py +++ b/waterbutler/providers/s3/metadata.py @@ -9,6 +9,11 @@ class S3Metadata(metadata.BaseMetadata): def provider(self): return 's3' + # S3 is a path-based provider and uses the default ``revalidate_path()``. + @property + def id(self): + return self.path + @property def name(self): return os.path.split(self.path)[1]