Mulled support.

Implement option to build mulled containers for tools:

```
    planemo mull [/path/to/tools]*
```

Implement option to force tool serving and execution to occur in mulled containers.

```
    planemo test --mulled_containers
```

This includes fixes for support or tools in Docker, something that has been around since the early days but seemingly broken the whole time.

xref #583

Author: jmchilton

planemo/commands/cmd_mull.py

@@ -0,0 +1,33 @@
+"""Module describing the planemo ``mull`` command."""
+import click
+
+from galaxy.tools.deps.mulled.mulled_build import mull_targets
+from galaxy.tools.deps.mulled.util import build_target
+
+from planemo import options
+from planemo.cli import command_function
+from planemo.conda import collect_conda_target_lists
+from planemo.mulled import build_mull_target_kwds
+
+
+@click.command('mull')
+@options.optional_tools_arg(multiple=True)
+@options.recursive_option()
+@command_function
+def cli(ctx, paths, **kwds):
+    """Build containers for specified tools.
+
+    Supplied tools will be inspected for referenced requirement packages. For
+    each combination of requirements a "mulled" container will be built. Galaxy
+    can automatically discover this container and subsequently use it to run
+    or test the tool.
+
+    For this to work, the tool's requirements will need to be present in a known
+    Conda channel such as bioconda (https://github.com/bioconda/bioconda-recipes).
+    This can be verified by running ``planemo lint --conda_requirements`` on the
+    target tool(s).
+    """
+    for conda_targets in collect_conda_target_lists(ctx, paths):
+        mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
+        mull_target_kwds = build_mull_target_kwds(ctx, **kwds)
+        mull_targets(mulled_targets, command="build", **mull_target_kwds)

planemo/conda.py

@@ -44,6 +44,20 @@ def collect_conda_targets(ctx, paths, found_tool_callback=None, conda_context=No
     return conda_targets
 
 
+def collect_conda_target_lists(ctx, paths, found_tool_callback=None):
+    """Load CondaTarget lists from supplied artifact sources.
+
+    If a tool contains more than one requirement, the requirements will all
+    appear together as one list element of the output list.
+    """
+    conda_target_lists = set([])
+    for (tool_path, tool_source) in yield_tool_sources_on_paths(ctx, paths):
+        if found_tool_callback:
+            found_tool_callback(tool_path)
+        conda_target_lists.add(frozenset(tool_source_conda_targets(tool_source)))
+    return conda_target_lists
+
+
 def tool_source_conda_targets(tool_source):
     """Load CondaTarget object from supplied abstract tool source."""
     requirements, _ = tool_source.parse_requirements_and_containers()
@@ -53,5 +67,6 @@ def tool_source_conda_targets(tool_source):
 __all__ = [
     "build_conda_context",
     "collect_conda_targets",
+    "collect_conda_target_lists",
     "tool_source_conda_targets",
 ]

planemo/galaxy/config.py

@@ -22,6 +22,7 @@
 
 from planemo import git
 from planemo.conda import build_conda_context
+from planemo.config import OptionSource
 from planemo.docker import docker_host_args
 from planemo.io import (
     communicate,
@@ -32,6 +33,7 @@
     warn,
     write_file,
 )
+from planemo.mulled import build_involucro_context
 from planemo.shed import tool_shed_url
 
 from .api import (
@@ -118,13 +120,20 @@
     </handlers>
     <destinations default="planemo_dest">
         <destination id="planemo_dest" runner="planemo_runner">
-            <param id="docker_enable">${docker_enable}</param>
+            <param id="require_container">${require_container}</param>
+            <param id="docker_enabled">${docker_enable}</param>
             <param id="docker_sudo">${docker_sudo}</param>
             <param id="docker_sudo_cmd">${docker_sudo_cmd}</param>
             <param id="docker_cmd">${docker_cmd}</param>
-            <param id="docker_host">${docker_host}</param>
+            ${docker_host_param}
+        </destination>
+        <destination id="upload_dest" runner="planemo_runner">
+            <param id="docker_enable">false</param>
         </destination>
     </destinations>
+    <tools>
+        <tool id="upload1" destination="upload_dest" />
+    </tools>
 </job_conf>
 """
 
@@ -211,6 +220,7 @@ def config_join(*args):
         )
         port = _get_port(kwds)
         properties = _shared_galaxy_properties(kwds)
+        _handle_container_resolution(ctx, kwds, properties)
         master_api_key = _get_master_api_key(kwds)
 
         template_args = dict(
@@ -276,6 +286,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
     )
     galaxy_root = _check_galaxy(ctx, **kwds)
     install_galaxy = galaxy_root is None
+
+    # Duplicate block in docker variant above.
+    if kwds.get("mulled_containers", False) and not kwds.get("docker", False):
+        if ctx.get_option_source("docker") != OptionSource.cli:
+            kwds["docker"] = True
+        else:
+            raise Exception("Specified no docker and mulled containers together.")
+
     with _config_directory(ctx, **kwds) as config_directory:
         def config_join(*args):
             return os.path.join(config_directory, *args)
@@ -374,6 +392,7 @@ def config_join(*args):
             migrated_tools_config=empty_tool_conf,
             test_data_dir=test_data_dir,  # TODO: make gx respect this
         ))
+        _handle_container_resolution(ctx, kwds, properties)
         if not for_tests:
             properties["database_connection"] = _database_connection(database_location, **kwds)
 
@@ -1106,13 +1125,20 @@ def _handle_job_config_file(config_directory, server_name, kwds):
             config_directory,
             "job_conf.xml",
         )
+        docker_enable = str(kwds.get("docker", False))
+        docker_host = str(kwds.get("docker_host", docker_util.DEFAULT_HOST))
+        docker_host_param = ""
+        if docker_host:
+            docker_host_param = """<param id="docker_host">%s</param>""" % docker_host
+
         conf_contents = Template(template_str).safe_substitute({
             "server_name": server_name,
             "docker_enable": str(kwds.get("docker", False)),
+            "require_container": docker_enable,
             "docker_sudo": str(kwds.get("docker_sudo", False)),
             "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
             "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
-            "docker_host": str(kwds.get("docker_host", docker_util.DEFAULT_HOST)),
+            "docker_host": docker_host_param,
         })
         write_file(job_config_file, conf_contents)
     kwds["job_config_file"] = job_config_file
@@ -1184,6 +1210,14 @@ def add_attribute(key, value):
             kwds["dependency_resolvers_config_file"] = resolvers_conf
 
 
+def _handle_container_resolution(ctx, kwds, galaxy_properties):
+    if kwds.get("mulled_containers", False):
+        galaxy_properties["enable_beta_mulled_containers"] = "True"
+        involucro_context = build_involucro_context(ctx, **kwds)
+        galaxy_properties["involucro_auto_init"] = "False"  # Use planemo's
+        galaxy_properties["involucro_path"] = involucro_context.involucro_bin
+
+
 def _handle_job_metrics(config_directory, kwds):
     metrics_conf = os.path.join(config_directory, "job_metrics_conf.xml")
     open(metrics_conf, "w").write(EMPTY_JOB_METRICS_TEMPLATE)

planemo/mulled.py

@@ -0,0 +1,46 @@
+"""Planemo specific utilities for dealing with mulled containers.
+
+The extend Galaxy/galaxy-lib's features with planemo specific idioms.
+"""
+from __future__ import absolute_import
+
+import os
+
+from galaxy.tools.deps.mulled.mulled_build import (
+    DEFAULT_CHANNELS,
+    ensure_installed,
+    InvolucroContext,
+)
+
+from planemo.io import shell
+
+
+def build_involucro_context(ctx, **kwds):
+    """Build a galaxy-lib CondaContext tailored to planemo use.
+
+    Using planemo's common command-line/global config options.
+    """
+    involucro_path_default = os.path.join(ctx.workspace, "involucro")
+    involucro_path = kwds.get("involucro_path", involucro_path_default)
+    use_planemo_shell = kwds.get("use_planemo_shell_exec", True)
+    shell_exec = shell if use_planemo_shell else None
+    involucro_context = InvolucroContext(involucro_bin=involucro_path,
+                                         shell_exec=shell_exec)
+    if not ensure_installed(involucro_context, True):
+        raise Exception("Failed to install involucro for Planemo.")
+    return involucro_context
+
+
+def build_mull_target_kwds(ctx, **kwds):
+    """Adapt Planemo's CLI and workspace configuration to galaxy-lib's mulled_build options."""
+    involucro_context = build_involucro_context(ctx, **kwds)
+    channels = kwds.get("conda_ensure_channels", ",".join(DEFAULT_CHANNELS))
+
+    return {
+        'involucro_context': involucro_context,
+        'channels': channels.split(","),
+    }
+
+__all__ = [
+    "build_involucro_context",
+]

planemo/options.py

@@ -333,6 +333,14 @@ def job_config_option():
     )
 
 
+def mulled_containers_option():
+    return planemo_option(
+        "--mulled_containers",
+        is_flag=True,
+        help="Test tools against mulled containers (forces --docker).",
+    )
+
+
 def install_galaxy_option():
     return planemo_option(
         "--install_galaxy",
@@ -430,6 +438,8 @@ def conda_debug_option():
 
 def conda_ensure_channels_option():
     return planemo_option(
+        "conda_ensure_channels",
+        "--conda_channels",
         "--conda_ensure_channels",
         type=str,
         use_global_config=True,
@@ -878,6 +888,7 @@ def galaxy_target_options():
         no_cleanup_option(),
         galaxy_email_option(),
         galaxy_docker_options(),
+        mulled_containers_option(),
         # Profile options...
         job_config_option(),
         tool_dependency_dir_option(),

requirements.txt

@@ -10,6 +10,6 @@ virtualenv
 lxml
 gxformat2>=0.1.1
 ephemeris>=0.2.0
-galaxy-lib>=16.10.3
+galaxy-lib>=16.10.5
 html5lib>=0.9999999,!=0.99999999,!=0.999999999,!=1.0b10,!=1.0b09 ; python_version == '2.7'
 cwltool==1.0.20160726135535 ; python_version == '2.7'