feat: do not create dataset implicitly (#779)

* feat: do not create dataset implicitly

* refactor: remove dataset overwrite prompt

Author: mohammad-alisafaee

conftest.py

@@ -176,7 +176,7 @@ def client(project):
 @pytest.fixture
 def dataset(client):
     """Create a dataset."""
-    with client.with_dataset(name='dataset') as dataset:
+    with client.with_dataset(name='dataset', create=True) as dataset:
         dataset.creator = [{
             'affiliation': 'xxx',
             'email': 'me@example.com',

renku/cli/dataset.py

@@ -58,6 +58,13 @@
 This will copy the contents of ``data-url`` to the dataset and add it
 to the dataset metadata.
 
+You can create a dataset when you add data to it for the first time by passing
+``--create`` flag to add command:
+
+.. code-block:: console
+
+    $ renku dataset add --create new-dataset http://data-url
+
 To add data from a git repository, you can specify it via https or git+ssh
 URL schemes. For example,
 
@@ -231,16 +238,6 @@
 from renku.core.errors import DatasetNotFound, InvalidAccessToken
 
 
-def prompt_duplicate_dataset(existing_dataset):
-    """Check if existing dataset should be overwritten.
-
-    :return: True if user confirmed overwriting.
-    """
-    warn_ = WARNING + 'This dataset already exists.'
-    click.echo(warn_)
-    return click.confirm('Do you wish to overwrite it?', abort=True)
-
-
 def prompt_access_token(exporter):
     """Prompt user for an access token for a provider.
 
@@ -365,7 +362,7 @@ def dataset(ctx, revision, datadir, format):
 @click.argument('name')
 def create(name):
     """Create an empty dataset in the current repo."""
-    create_dataset(name, handle_duplicate_fn=prompt_duplicate_dataset)
+    create_dataset(name)
     click.secho('OK', fg='green')
 
 
@@ -387,6 +384,9 @@ def edit(dataset_id):
 @click.option(
     '--force', is_flag=True, help='Allow adding otherwise ignored files.'
 )
+@click.option(
+    '--create', is_flag=True, help='Create dataset if it does not exist.'
+)
 @click.option(
     '-s',
     '--src',
@@ -404,14 +404,15 @@ def edit(dataset_id):
     default='',
     help='Destination file or directory within the dataset path'
 )
-def add(name, urls, link, force, sources, destination):
+def add(name, urls, link, force, create, sources, destination):
     """Add data to a dataset."""
     progress = partial(progressbar, label='Adding data to dataset')
     add_file(
         urls=urls,
         name=name,
         link=link,
         force=force,
+        create=create,
         sources=sources,
         destination=destination,
         urlscontext=progress
@@ -580,10 +581,7 @@ def export_(id, provider, publish, tag):
     is_flag=True,
     help='Extract files before importing to dataset.'
 )
-@click.option(
-    '-y', '--yes', is_flag=True, help='Confirm unlinking of all files.'
-)
-def import_(uri, name, extract, yes):
+def import_(uri, name, extract):
     """Import data from a 3rd party provider.
 
     Supported providers: [Zenodo, Dataverse]
@@ -612,11 +610,9 @@ def _init(lock, id_queue):
         name,
         extract,
         with_prompt=True,
-        handle_duplicate_fn=prompt_duplicate_dataset,
         pool_init_fn=_init,
         pool_init_args=(mp.RLock(), id_queue),
-        download_file_fn=download_file_with_progress,
-        force=yes
+        download_file_fn=download_file_with_progress
     )
     click.secho('OK', fg='green')
 

renku/core/commands/dataset.py

@@ -100,17 +100,15 @@ def dataset_parent(client, revision, datadir, format, ctx=None):
 @pass_local_client(
     clean=False, commit=True, commit_only=DATASET_METADATA_PATHS
 )
-def create_dataset(client, name, handle_duplicate_fn=None):
+def create_dataset(client, name):
     """Create an empty dataset in the current repo.
 
     :raises: ``renku.core.errors.ParameterError``
     """
-    existing = client.load_dataset(name=name)
-    if (not existing or handle_duplicate_fn and handle_duplicate_fn(existing)):
-        with client.with_dataset(name=name) as dataset:
-            creator = Creator.from_git(client.repo)
-            if creator not in dataset.creator:
-                dataset.creator.append(creator)
+    with client.with_dataset(name=name, create=True) as dataset:
+        creator = Creator.from_git(client.repo)
+        if creator not in dataset.creator:
+            dataset.creator.append(creator)
 
 
 @pass_local_client(
@@ -140,15 +138,16 @@ def add_file(
     name,
     link=False,
     force=False,
+    create=False,
     sources=(),
     destination='',
     with_metadata=None,
     urlscontext=contextlib.nullcontext
 ):
     """Add data file to a dataset."""
     add_to_dataset(
-        client, urls, name, link, force, sources, destination, with_metadata,
-        urlscontext
+        client, urls, name, link, force, create, sources, destination,
+        with_metadata, urlscontext
     )
 
 
@@ -158,6 +157,7 @@ def add_to_dataset(
     name,
     link=False,
     force=False,
+    create=False,
     sources=(),
     destination='',
     with_metadata=None,
@@ -169,7 +169,9 @@ def add_to_dataset(
         with_metadata.identifier
     ) if with_metadata else None
     try:
-        with client.with_dataset(name=name, identifier=identifier) as dataset:
+        with client.with_dataset(
+            name=name, identifier=identifier, create=create
+        ) as dataset:
             with urlscontext(urls) as bar:
                 client.add_data_to_dataset(
                     dataset,
@@ -195,6 +197,13 @@ def add_to_dataset(
 
                 dataset.update_metadata(with_metadata)
 
+    except DatasetNotFound:
+        raise DatasetNotFound(
+            'Dataset "{0}" does not exist.\n'
+            'Use "renku dataset create {0}" to create the dataset or retry '
+            '"renku dataset add {0}" command with "--create" option for '
+            'automatic dataset creation.'.format(name)
+        )
     except (FileNotFoundError, git.exc.NoSuchPathError):
         raise ParameterError('Could not process \n{0}'.format('\n'.join(urls)))
 
@@ -393,8 +402,6 @@ def import_dataset(
     name,
     extract,
     with_prompt=False,
-    force=False,
-    handle_duplicate_fn=None,
     pool_init_fn=None,
     pool_init_args=None,
     download_file_fn=default_download_file
@@ -428,8 +435,7 @@ def import_dataset(
                     record.links.get('latest_html')
                 ) + text_prompt
 
-            if not force:
-                click.confirm(text_prompt, abort=True)
+            click.confirm(text_prompt, abort=True)
 
     except KeyError as e:
         raise ParameterError((
@@ -482,27 +488,20 @@ def import_dataset(
         pool.close()
 
         dataset_name = name or dataset.display_name
-        existing = client.load_dataset(name=dataset_name)
-        if (
-            not existing or force or
-            (handle_duplicate_fn and handle_duplicate_fn(dataset_name))
-        ):
-            add_to_dataset(
-                client,
-                urls=[str(p) for p in Path(data_folder).glob('*')],
-                name=dataset_name,
-                with_metadata=dataset
-            )
+        add_to_dataset(
+            client,
+            urls=[str(p) for p in Path(data_folder).glob('*')],
+            name=dataset_name,
+            with_metadata=dataset,
+            create=True
+        )
 
-            if dataset.version:
-                tag_name = re.sub('[^a-zA-Z0-9.-_]', '_', dataset.version)
-                tag_dataset(
-                    client,
-                    dataset_name,
-                    tag_name,
-                    'Tag {} created by renku import'.format(dataset.version),
-                    force=True
-                )
+        if dataset.version:
+            tag_name = re.sub('[^a-zA-Z0-9.-_]', '_', dataset.version)
+            tag_dataset(
+                client, dataset_name, tag_name,
+                'Tag {} created by renku import'.format(dataset.version)
+            )
 
 
 @pass_local_client(clean=True, commit=True, commit_only=DATASET_METADATA_PATHS)

renku/core/errors.py

@@ -293,10 +293,8 @@ def __init__(self, returncode, success_codes=None):
 class DatasetNotFound(RenkuException):
     """Raise when dataset is not found."""
 
-    def __init__(self):
+    def __init__(self, msg='Dataset is not found.'):
         """Build a custom message."""
-        msg = 'Dataset is not found.'
-
         super(DatasetNotFound, self).__init__(msg)
 
 

renku/core/management/datasets.py

@@ -109,7 +109,7 @@ def load_dataset(self, name=None):
                 return self.get_dataset(path)
 
     @contextmanager
-    def with_dataset(self, name=None, identifier=None):
+    def with_dataset(self, name=None, identifier=None, create=False):
         """Yield an editable metadata object for a dataset."""
         dataset = self.load_dataset(name=name)
         clean_up_required = False
@@ -121,6 +121,8 @@ def with_dataset(self, name=None, identifier=None):
                     'Dataset name {} is not valid.'.format(name)
                 )
 
+            if not create:
+                raise errors.DatasetNotFound
             clean_up_required = True
             dataset_ref = None
             identifier = str(uuid.uuid4())
@@ -143,6 +145,11 @@ def with_dataset(self, name=None, identifier=None):
                 )
                 dataset_ref.set_reference(path)
 
+        elif create:
+            raise errors.DatasetExistsError(
+                'Dataset exists: "{}".'.format(name)
+            )
+
         dataset_path = self.path / self.datadir / dataset.name
         dataset_path.mkdir(parents=True, exist_ok=True)