Overhaul Galaxy job config handling.

- More options from the command-line - multiple runners, TPV shared database, singularity, etc...
- Switch from job conf via XML to job conf via YAML.
- Bring in dependency on galaxy-job-config-init to handle a lot of that logic.
- Commands to assign binding a job config to a created Planemo profile.

Author: jmchilton

planemo/commands/cmd_config_init.py

@@ -12,6 +12,7 @@
 from planemo.cli import command_function
 from planemo.io import (
     info,
+    launch_if_open_flagged,
     warn,
 )
 
@@ -44,6 +45,7 @@
 
 @click.command("config_init")
 @options.optional_project_arg(exists=None)
+@options.open_file_option()
 @click.option("--template", default=None)
 @command_function
 def cli(ctx, path, template=None, **kwds):
@@ -59,3 +61,4 @@ def cli(ctx, path, template=None, **kwds):
     with open(config_path, "w") as f:
         f.write(CONFIG_TEMPLATE)
         info(SUCCESS_MESSAGE % config_path)
+    launch_if_open_flagged(config_path, **kwds)

planemo/commands/cmd_dockstore_init.py

@@ -6,6 +6,7 @@
 
 from planemo import options
 from planemo.cli import command_function
+from planemo.io import launch_if_open_flagged
 from planemo.workflow_lint import (
     DOCKSTORE_REGISTRY_CONF,
     generate_dockstore_yaml,
@@ -15,6 +16,7 @@
 @click.command("dockstore_init")
 @options.optional_project_arg()
 @options.publish_dockstore_option()
+@options.open_file_option()
 @command_function
 def cli(ctx, path=".", **kwds):
     """Initialize a .dockstore.yml configuration file for workflows in directory.
@@ -32,3 +34,4 @@ def cli(ctx, path=".", **kwds):
     contents = generate_dockstore_yaml(path, kwds["publish"])
     with open(dockstore_path, "w") as f:
         f.write(contents)
+    launch_if_open_flagged(dockstore_path, **kwds)

planemo/commands/cmd_job_config_init.py

@@ -0,0 +1,48 @@
+"""Module describing the planemo ``project_init`` command."""
+
+import os
+import sys
+
+import click
+from gxjobconfinit import build_job_config, ConfigArgs
+from gxjobconfinit.generate import DevelopmentContext
+
+from planemo import options
+from planemo.galaxy.config import get_all_tool_path_from_kwds
+from planemo.cli import command_function
+from planemo.io import (
+    info,
+    launch_if_open_flagged,
+    warn,
+)
+from planemo.runnable import for_paths
+from planemo.tools import uris_to_paths
+
+SUCCESS_MESSAGE = "Wrote configuration template to %s, please open with editor to customize if needed."
+
+
+@click.command("job_config_init")
+@options.optional_tools_arg(multiple=True, allow_uris=True)
+@options.open_file_option()
+@options.job_config_init_options()
+@command_function
+def cli(ctx, uris, **kwds):
+    """Initialize an small Galaxy job config file for hosted workflow runs."""
+    config_path = "job_conf.yml"
+    if os.path.exists(config_path):
+        warn(f"File '{config_path}' already exists, exiting.")
+        sys.exit(1)
+
+    paths = uris_to_paths(ctx, uris)
+    runnables = for_paths(paths)
+    tool_paths = get_all_tool_path_from_kwds(runnables, **kwds)
+    dev_context = DevelopmentContext(
+        kwds.get("test_data", None),
+        tool_paths,
+    )
+    init_config = ConfigArgs.from_dict(**kwds)
+    job_config = build_job_config(init_config, dev_context)
+    with open(config_path, "w") as f:
+        f.write(job_config)
+        info(SUCCESS_MESSAGE % config_path)
+    launch_if_open_flagged(config_path, **kwds)

planemo/commands/cmd_open.py

@@ -5,7 +5,7 @@
 from planemo.cli import command_function
 
 
-@click.command("docs")
+@click.command("open")
 @click.argument(
     "path",
     metavar="PATH",

planemo/commands/cmd_profile_job_config_init.py

@@ -0,0 +1,30 @@
+"""Module describing the planemo ``project_init`` command."""
+
+import click
+
+from planemo import options
+from planemo.galaxy import profiles
+from planemo.cli import command_function
+from planemo.io import (
+    info,
+    launch_if_open_flagged,
+    warn,
+)
+
+SUCCESS_MESSAGE = "Wrote configuration template to %s for your profile, please open with editor to customize if needed."
+
+
+@click.command("profile_job_config_init")
+@options.profile_name_argument()
+@options.open_file_option()
+@options.job_config_init_options()
+@command_function
+def cli(ctx, profile_name, **kwds):
+    """Initialize a Galaxy job configuration for specified profile."""
+    if not profiles.profile_exists(ctx, profile_name, **kwds):
+        warn(f"Profile '{profile_name}' does not exist, exiting.")
+        pass
+
+    config_path = profiles.initialize_job_config(ctx, profile_name, **kwds)
+    info(SUCCESS_MESSAGE % config_path)
+    launch_if_open_flagged(config_path, **kwds)

planemo/galaxy/config.py

@@ -19,15 +19,20 @@
 from typing import (
     Any,
     Dict,
-    Iterable,
     List,
     Optional,
     Set,
     TYPE_CHECKING,
 )
 
+from gxjobconfinit.generate import (
+    build_job_config,
+    ConfigArgs,
+    create_docker_volumes,
+)
+from gxjobconfinit.generate import DevelopmentContext
+
 from galaxy.tool_util.deps import docker_util
-from galaxy.tool_util.deps.container_volumes import DockerVolume
 from galaxy.util.commands import argv_to_str
 from packaging.version import parse as parse_version
 
@@ -122,33 +127,6 @@
 </tool_sheds>
 """
 
-JOB_CONFIG_LOCAL = """<job_conf>
-    <plugins>
-        <plugin id="planemo_runner" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
-    </plugins>
-    <handlers>
-    </handlers>
-    <destinations default="planemo_dest">
-        <destination id="planemo_dest" runner="planemo_runner">
-            <param id="require_container">${require_container}</param>
-            <param id="docker_enabled">${docker_enable}</param>
-            <param id="docker_sudo">${docker_sudo}</param>
-            <param id="docker_sudo_cmd">${docker_sudo_cmd}</param>
-            <param id="docker_cmd">${docker_cmd}</param>
-            <param id="docker_volumes">${docker_volumes}</param>
-            <param id="docker_run_extra_arguments"><![CDATA[${docker_run_extra_arguments}]]></param>
-            ${docker_host_param}
-        </destination>
-        <destination id="upload_dest" runner="planemo_runner">
-            <param id="docker_enabled">false</param>
-        </destination>
-    </destinations>
-    <tools>
-        <tool id="upload1" destination="upload_dest" />
-    </tools>
-</job_conf>
-"""
-
 REFGENIE_CONFIG_TEMPLATE = """
 config_version: %s
 genome_folder: '%s'
@@ -216,22 +194,6 @@ def read_log(ctx, log_path, e: threading.Event):
             log_fh.close()
 
 
-def create_docker_volumes(paths: Iterable[str]) -> Iterable[DockerVolume]:
-    """
-    Creates string of the format "host_path:target_path:mode" and deduplicates overlapping mounts.
-    """
-    docker_volumes: Dict[str, DockerVolume] = {}
-    for path in paths:
-        docker_volume = DockerVolume.from_str(path)
-        if docker_volume.path in docker_volumes:
-            # volume has been specified already, make sure we use "rw" if any of the modes are "rw"
-            if docker_volume.mode == "rw" or docker_volumes[docker_volume.path].mode == "rw":
-                docker_volumes[docker_volume.path].mode = "rw"
-        else:
-            docker_volumes[docker_volume.path] = docker_volume
-    return docker_volumes.values()
-
-
 @contextlib.contextmanager
 def docker_galaxy_config(ctx, runnables, for_tests=False, **kwds):
     """Set up a ``GalaxyConfig`` for Docker container."""
@@ -572,6 +534,12 @@ def get_refgenie_config(galaxy_root, refgenie_dir):
     return REFGENIE_CONFIG_TEMPLATE % (config_version, refgenie_dir)
 
 
+def get_all_tool_path_from_kwds(runnables: List["Runnable"], **kwds) -> Set[str]:
+    galaxy_root = kwds.get("galaxy_root")
+    extra_tools = kwds.get("extra_tools")
+    return _all_tool_paths(runnables, galaxy_root, extra_tools)
+
+
 def _all_tool_paths(
     runnables: List["Runnable"], galaxy_root: Optional[str] = None, extra_tools: Optional[List[str]] = None
 ) -> Set[str]:
@@ -1378,40 +1346,15 @@ def _handle_job_config_file(
 ):
     job_config_file = kwds.get("job_config_file", None)
     if not job_config_file:
-        template_str = JOB_CONFIG_LOCAL
+        dev_context = DevelopmentContext(
+            test_data_dir,
+            all_tool_paths,
+        )
+        init_config = ConfigArgs.from_dict(**kwds)
+        conf_contents = build_job_config(init_config, dev_context)
         job_config_file = os.path.join(
             config_directory,
-            "job_conf.xml",
-        )
-        docker_enable = str(kwds.get("docker", False))
-        docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
-        docker_host_param = ""
-        if docker_host:
-            docker_host_param = f"""<param id="docker_host">{docker_host}</param>"""
-
-        volumes = list(kwds.get("docker_extra_volume") or [])
-        if test_data_dir:
-            volumes.append(f"{test_data_dir}:ro")
-
-        docker_volumes_str = "$defaults"
-        if volumes:
-            # exclude tool directories, these are mounted :ro by $defaults
-            all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths}
-            extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs)
-            docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}"
-
-        conf_contents = Template(template_str).safe_substitute(
-            {
-                "server_name": server_name,
-                "docker_enable": docker_enable,
-                "require_container": "false",
-                "docker_sudo": str(kwds.get("docker_sudo", False)),
-                "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
-                "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
-                "docker_host_param": docker_host_param,
-                "docker_volumes": docker_volumes_str,
-                "docker_run_extra_arguments": kwds.get("docker_run_extra_arguments", ""),
-            }
+            "job_conf.yml",
         )
         write_file(job_config_file, conf_contents)
     kwds["job_config_file"] = job_config_file

planemo/galaxy/profiles.py

@@ -9,6 +9,7 @@
 import shutil
 
 from galaxy.util.commands import which
+from gxjobconfinit import build_job_config, ConfigArgs
 
 from planemo.database import create_database_source
 from planemo.galaxy.api import test_credentials_valid
@@ -226,6 +227,24 @@ def _profile_directory(ctx, profile_name):
     return os.path.join(ctx.galaxy_profiles_directory, profile_name)
 
 
+def initialize_job_config(ctx, profile_name, **kwds):
+    profile_directory = _profile_directory(ctx, profile_name)
+    job_config_path = os.path.join(profile_directory, "job_conf.yml")
+    if os.path.exists(job_config_path):
+        raise Exception(f"File '{job_config_path}' already exists, exiting.")
+
+    init_config = ConfigArgs.from_dict(**kwds)
+    job_config = build_job_config(init_config)
+    with open(job_config_path, "w") as f:
+        f.write(job_config)
+
+    config, profile_config_path = _load_profile_to_json(ctx, profile_name)
+    config["job_config_file"] = os.path.abspath(profile_config_path)
+    with open(profile_config_path, "w") as f:
+        json.dump(config, f)
+    return job_config_path
+
+
 __all__ = (
     "create_profile",
     "delete_profile",

planemo/io.py

@@ -425,3 +425,8 @@ def coalesce_return_codes(ret_codes, assert_at_least_one=False):
         coalesced_return_code = 255 + coalesced_return_code
 
     return coalesced_return_code
+
+
+def launch_if_open_flagged(file, **kwd):
+    if kwd.get("open"):
+        click.launch(file)