Update shed_diff and shed_download for multi-repo-shed-yamls.

 - Both realize repositories for configuration.
 - Both can now take in -r/--recursive arguments.
 - Download will attempt to download many repositories at once.
 - shed_diff now operates properly on the realized files instead of the local directory.

Still need to do a "smart diff" that filters out toolshed and changeset_revision attributes before #141 can be closed.

Author: jmchilton

planemo/commands/cmd_shed_diff.py

@@ -3,6 +3,7 @@
 import os
 import shutil
 import tempfile
+import sys
 
 import click
 
@@ -30,6 +31,7 @@
          " this to testtoolshed.",
     default=None,
 )
+@options.recursive_shed_option()
 @pass_context
 def cli(ctx, path, **kwds):
     """Produce diff between local repository and Tool Shed contents.
@@ -50,14 +52,19 @@ def cli(ctx, path, **kwds):
         % planemo shed_diff --owner peterjc --name blast_rbh
             --shed_target_source testtoolshed
     """
-    working = tempfile.mkdtemp(prefix="tool_shed_diff_")
-    try:
-        diff_in(ctx, working, path, **kwds)
-    finally:
-        shutil.rmtree(working)
+    def diff(realized_repository):
+        working = tempfile.mkdtemp(prefix="tool_shed_diff_")
+        try:
+            diff_in(ctx, working, realized_repository, **kwds)
+        finally:
+            shutil.rmtree(working)
 
+    exit_code = shed.for_each_repository(diff, path, **kwds)
+    sys.exit(exit_code)
 
-def diff_in(ctx, working, path, **kwds):
+
+def diff_in(ctx, working, realized_repository, **kwds):
+    path = realized_repository.path
     shed_target_source = kwds.get("shed_target_source", None)
 
     label_a = "_%s_" % (shed_target_source if shed_target_source else "local")
@@ -73,20 +80,19 @@ def diff_in(ctx, working, path, **kwds):
     shed.download_tarball(
         ctx,
         tsi,
-        path,
+        realized_repository,
         destination=other,
         clean=True,
         **kwds
     )
-
     if shed_target_source:
         new_kwds = kwds.copy()
         new_kwds["shed_target"] = shed_target_source
         tsi = shed.tool_shed_client(ctx, read_only=True, **new_kwds)
         shed.download_tarball(
             ctx,
             tsi,
-            path,
+            realized_repository,
             destination=mine,
             clean=True,
             **new_kwds

planemo/commands/cmd_shed_download.py

@@ -1,5 +1,7 @@
 """
 """
+import sys
+
 import click
 
 from planemo.cli import pass_context
@@ -19,17 +21,26 @@
     '--destination',
     default="shed_download.tar.gz",
     type=target_path,
-    help="Destination of tarball to download - if this doesn't end in 'gz' it "
-         "will be treated as a directory to extract tool contents into"
-         "(defaults to shed_download.tar.gz)."
+    help="Destination pattern of tarball(s) to download - if this doesn't "
+         "end in 'gz' it will be treated as a directory to extract tool "
+         "contents into (defaults to shed_download.tar.gz). If multiple "
+         "repositories are discovered in a .shed.yml file these will be "
+         "created as shed_download_<name>.tar.gz by default for instance, "
+         "simpler repositories will just be downloaded to the specified file."
 )
 @options.shed_owner_option()
 @options.shed_name_option()
 @options.shed_target_option()
+@options.recursive_shed_option()
 @pass_context
 def cli(ctx, path, **kwds):
     """Download a tool repository as a tarball from the tool shed and extract
     to the specified directory.
     """
     tsi = shed.tool_shed_client(ctx, read_only=True, **kwds)
-    shed.download_tarball(ctx, tsi, path, **kwds)
+
+    def download(realized_repository):
+        return shed.download_tarball(ctx, tsi, realized_repository, **kwds)
+
+    exit_code = shed.for_each_repository(download, path, **kwds)
+    sys.exit(exit_code)

planemo/shed/__init__.py

@@ -150,11 +150,8 @@ def upload_repository(ctx, realized_repository, **kwds):
     if not tar_path:
         tar_path = build_tarball(path, **kwds)
     if kwds["tar_only"]:
-        suffix = ""
-        if realized_repository.multiple:
-            name = realized_repository.config["name"]
-            suffix = "_%s" % name.replace("-", "_")
-        shell("cp %s shed_upload%s.tar.gz" % (tar_path, suffix))
+        name = realized_repository.pattern_to_file_name("shed_upload.tar.gz")
+        shell("cp '%s' '%s'" % (tar_path, name))
         return 0
     tsi = tool_shed_client(ctx, **kwds)
     update_kwds = {}
@@ -377,9 +374,10 @@ def create_repository_for(ctx, tsi, name, repo_config):
     return repo
 
 
-def download_tarball(ctx, tsi, path, **kwds):
-    repo_id = find_repository_id(ctx, tsi, path, **kwds)
-    destination = kwds.get('destination', 'shed_download.tar.gz')
+def download_tarball(ctx, tsi, realized_repository, **kwds):
+    repo_id = realized_repository.find_repository_id(ctx, tsi)
+    destination_pattern = kwds.get('destination', 'shed_download.tar.gz')
+    destination = realized_repository.pattern_to_file_name(destination_pattern)
     to_directory = not destination.endswith("gz")
     download_tar(tsi, repo_id, destination, to_directory=to_directory)
     if to_directory:
@@ -778,6 +776,19 @@ def __init__(self, realized_path, real_path, config, multiple):
         self.name = config["name"]
         self.multiple = multiple
 
+    def pattern_to_file_name(self, pattern):
+        if not self.multiple:
+            return pattern
+
+        name = self.config["name"]
+        suffix = "_%s" % name.replace("-", "_")
+
+        if "." not in pattern:
+            return pattern + suffix
+        else:
+            parts = pattern.split(".", 1)
+            return parts[0] + suffix + "." + parts[1]
+
     def find_repository_id(self, ctx, tsi):
         try:
             repo_id = _find_repository_id(

tests/test_shed_download.py

@@ -0,0 +1,37 @@
+from os.path import join
+
+from .test_utils import CliShedTestCase
+from .test_shed_upload import assert_exists
+
+
+class ShedUploadTestCase(CliShedTestCase):
+
+    def test_download_expansion_tars(self):
+        with self._isolate_repo("multi_repos_flat_flag") as f:
+            upload_command = [
+                "shed_upload", "--force_repository_creation"
+            ]
+            upload_command.extend(self._shed_args())
+            self._check_exit_code(upload_command)
+            download_command = [
+                "shed_download",
+            ]
+            download_command.extend(self._shed_args(read_only=True))
+            self._check_exit_code(download_command)
+            assert_exists(join(f, "shed_download_cs_cat1.tar.gz"))
+            assert_exists(join(f, "shed_download_cs_cat2.tar.gz"))
+
+    def test_download_expansion_dir(self):
+        with self._isolate_repo("multi_repos_flat_flag") as f:
+            upload_command = [
+                "shed_upload", "--force_repository_creation"
+            ]
+            upload_command.extend(self._shed_args())
+            self._check_exit_code(upload_command)
+            download_command = [
+                "shed_download", "--destination", "shed_d"
+            ]
+            download_command.extend(self._shed_args(read_only=True))
+            self._check_exit_code(download_command)
+            assert_exists(join(f, "shed_d_cs_cat1", "cat1.xml"))
+            assert_exists(join(f, "shed_d_cs_cat2", "cat2.xml"))

tests/test_shed_upload.py

@@ -16,7 +16,7 @@ def test_tar_single(self):
             upload_command = ["shed_upload", "--tar_only"]
             upload_command.extend(self._shed_args())
             self._check_exit_code(upload_command)
-            _assert_exists(join(f, "shed_upload.tar.gz"))
+            assert_exists(join(f, "shed_upload.tar.gz"))
 
     def test_upload_not_exists(self):
         with self._isolate_repo("single_tool"):
@@ -68,7 +68,7 @@ def test_upload_filters_invalid_suite(self):
             self._check_exit_code(upload_command)
             target = self._untar(f, "shed_upload.tar.gz")
             # Only one file was in archive
-            _assert_exists(join(target, "repository_dependencies.xml"))
+            assert_exists(join(target, "repository_dependencies.xml"))
             # this got filtered
             assert not exists(join(target, "README.rst"))
 
@@ -79,7 +79,7 @@ def test_upload_suite_auto(self):
             self._check_exit_code(upload_command)
             target = self._untar(f, "shed_upload_suite_1.tar.gz")
             # Only one file was in archive
-            _assert_exists(join(target, "repository_dependencies.xml"))
+            assert_exists(join(target, "repository_dependencies.xml"))
 
     def test_upload_filters_ignore(self):
         with self._isolate_repo("single_tool_exclude") as f:
@@ -120,7 +120,7 @@ def test_upload_filters_invalid_package(self):
             self._check_exit_code(upload_command)
             target = self._untar(f, "shed_upload.tar.gz")
             # Only one file was in archive
-            _assert_exists(join(target, "tool_dependencies.xml"))
+            assert_exists(join(target, "tool_dependencies.xml"))
             # this got filtered
             assert not exists(join(target, "README.rst"))
             # .shed.yml always gets filtered
@@ -133,8 +133,8 @@ def test_upload_not_filters_unrestricted(self):
             self._check_exit_code(upload_command)
             target = self._untar(f, "shed_upload.tar.gz")
             # Only one file was in archive
-            _assert_exists(join(target, "repository_dependencies.xml"))
-            _assert_exists(join(target, "README.rst"))
+            assert_exists(join(target, "repository_dependencies.xml"))
+            assert_exists(join(target, "README.rst"))
 
     def test_upload_expansion_configured(self):
         with self._isolate_repo("multi_repos_flat_configured") as f:
@@ -205,15 +205,15 @@ def _verify_single_uploaded(self, f):
     def _verify_upload(self, f, download_files=[], download_args=[]):
         target = self._download_repo(f, download_args)
         for download_file in download_files:
-            _assert_exists(join(target, download_file))
+            assert_exists(join(target, download_file))
         return target
 
     def _check_tar(self, f, tar_path, contains=[], not_contains=[]):
         tar_path = join(f, tar_path)
-        _assert_exists(tar_path)
+        assert_exists(tar_path)
         target = self._untar(f, tar_path)
         for path in contains:
-            _assert_exists(join(target, path))
+            assert_exists(join(target, path))
         for path in not_contains:
             assert not exists(join(target, path))
         return target
@@ -224,7 +224,7 @@ def _download_repo(self, f, download_args=[]):
         download_command.extend(self._shed_args(read_only=True))
         self._check_exit_code(download_command)
         download = join(f, "shed_download.tar.gz")
-        _assert_exists(download)
+        assert_exists(download)
         return self._untar(f, "shed_download.tar.gz", tarbomb=False)
 
     def _untar(self, f, path, tarbomb=True):
@@ -247,7 +247,7 @@ def _untar(self, f, path, tarbomb=True):
             return target
 
 
-def _assert_exists(path):
+def assert_exists(path):
     dir_path = os.path.dirname(path)
     msg = None
     if not exists(dir_path):