Support for distro tools in workflow testing.

If tool ids from Galaxy's default tool panel are encountered in workflows being executed, served, or tested - they will be added to the tool conf generated for the Galaxy instance started by Planemo.

Fixes #780.

Author: jmchilton

planemo/galaxy/config.py

@@ -41,12 +41,16 @@
     gi,
     user_api_key,
 )
+from .distro_tools import (
+    DISTRO_TOOLS_ID_TO_PATH
+)
 from .run import (
     DOWNLOAD_GALAXY,
     setup_common_startup_args,
     setup_venv,
 )
 from .workflows import (
+    find_tool_ids,
     import_workflow,
     install_shed_repos,
 )
@@ -267,12 +271,13 @@ def config_join(*args):
         _handle_job_metrics(config_directory, kwds)
 
         shed_tool_conf = "config/shed_tool_conf.xml"
-        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+        all_tool_paths = _all_tool_paths(runnables, **kwds)
 
         tool_directories = set([])  # Things to mount...
         for tool_path in all_tool_paths:
             directory = os.path.dirname(os.path.normpath(tool_path))
-            tool_directories.add(directory)
+            if os.path.exists(directory):
+                tool_directories.add(directory)
 
         # TODO: remap these.
         tool_volumes = []
@@ -396,7 +401,7 @@ def config_join(*args):
         _ensure_directory(tool_dependency_dir)
 
         shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
-        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+        all_tool_paths = _all_tool_paths(runnables, **kwds)
         tool_definition = _tool_conf_entry_for(all_tool_paths)
         empty_tool_conf = config_join("empty_tool_conf.xml")
 
@@ -536,6 +541,19 @@ def config_join(*args):
         )
 
 
+def _all_tool_paths(runnables, **kwds):
+    tool_paths = [r.path for r in runnables if r.has_tools]
+    all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+    for runnable in runnables:
+        if runnable.type.name == "galaxy_workflow":
+            tool_ids = find_tool_ids(runnable.path)
+            for tool_id in tool_ids:
+                if tool_id in DISTRO_TOOLS_ID_TO_PATH:
+                    all_tool_paths.append(DISTRO_TOOLS_ID_TO_PATH[tool_id])
+
+    return all_tool_paths
+
+
 def _shared_galaxy_properties(config_directory, kwds, for_tests):
     """Setup properties useful for local and Docker Galaxy instances.
 

planemo/galaxy/distro_tools.py

@@ -0,0 +1,99 @@
+# file auto generated with scripts/tool_index_to_id_map.py
+DISTRO_TOOLS_ID_TO_PATH = {
+    "ucsc_table_direct1": "data_source/ucsc_tablebrowser.xml",
+    "MAF_To_Fasta1": "maf/maf_to_fasta.xml",
+    "ratmine": "data_source/ratmine.xml",
+    "mousemine": "data_source/mousemine.xml",
+    "__SORTLIST__": "${model_tools_path}/sort_collection_list.xml",
+    "__UNZIP_COLLECTION__": "${model_tools_path}/unzip_collection.xml",
+    "cbi_rice_mart": "data_source/cbi_rice_mart.xml",
+    "ucsc_table_direct_archaea1": "data_source/ucsc_tablebrowser_archaea.xml",
+    "wig_to_bigWig": "filters/wig_to_bigwig.xml",
+    "ebi_sra_main": "data_source/ebi_sra.xml",
+    "metabolicmine": "data_source/metabolicmine.xml",
+    "Extract_features1": "filters/gff/extract_GFF_Features.xml",
+    "upload1": "data_source/upload.xml",
+    "wc_gnu": "filters/wc_gnu.xml",
+    "join1": "filters/joiner.xml",
+    "random_lines1": "filters/randomlines.xml",
+    "modENCODEfly": "data_source/fly_modencode.xml",
+    "gff_filter_by_attribute": "filters/gff/gff_filter_by_attribute.xml",
+    "gtf2bedgraph": "filters/gtf2bedgraph.xml",
+    "ChangeCase": "filters/changeCase.xml",
+    "__FLATTEN__": "${model_tools_path}/flatten_collection.xml",
+    "wiggle2simple1": "filters/wiggle_to_simple.xml",
+    "GeneBed_Maf_Fasta2": "maf/genebed_maf_to_fasta.xml",
+    "trimmer": "filters/trimmer.xml",
+    "createInterval": "filters/CreateInterval.xml",
+    "gff_filter_by_feature_count": "filters/gff/gff_filter_by_feature_count.xml",
+    "Interval2Maf1": "maf/interval2maf.xml",
+    "genomespace_exporter": "genomespace/genomespace_exporter.xml",
+    "Show tail1": "filters/tailWrapper.xml",
+    "barchart_gnuplot": "plotting/bar_chart.xml",
+    "microbial_import1": "data_source/microbial_import.xml",
+    "axt_to_concat_fasta": "filters/axt_to_concat_fasta.xml",
+    "__FILTER_FROM_FILE__": "${model_tools_path}/filter_from_file.xml",
+    "Interval2Maf_pairwise1": "maf/interval2maf_pairwise.xml",
+    "Show beginning1": "filters/headWrapper.xml",
+    "axt_to_lav_1": "filters/axt_to_lav.xml",
+    "modmine": "data_source/modmine.xml",
+    "__EXPORT_HISTORY__": "${model_tools_path}/imp_exp/exp_history_to_archive.xml",
+    "MAF_Thread_For_Species1": "maf/maf_thread_for_species.xml",
+    "vcf_to_maf_customtrack1": "maf/vcf_to_maf_customtrack.xml",
+    "MAF_To_BED1": "maf/maf_to_bed.xml",
+    "__DATA_FETCH__": "${model_tools_path}/data_fetch.xml",
+    "__IMPORT_HISTORY__": "${model_tools_path}/imp_exp/imp_history_from_archive.xml",
+    "biomart": "data_source/biomart.xml",
+    "Sff_extractor": "filters/sff_extractor.xml",
+    "secure_hash_message_digest": "filters/secure_hash_message_digest.xml",
+    "MAF_Reverse_Complement_1": "maf/maf_reverse_complement.xml",
+    "mergeCols1": "filters/mergeCols.xml",
+    "gff2bed1": "filters/gff2bed.xml",
+    "Grouping1": "stats/grouping.xml",
+    "maf_limit_size1": "maf/maf_limit_size.xml",
+    "sort1": "filters/sorter.xml",
+    "Convert characters1": "filters/convert_characters.xml",
+    "MAF_To_Interval1": "maf/maf_to_interval.xml",
+    "MAF_filter": "maf/maf_filter.xml",
+    "MAF_split_blocks_by_species1": "maf/maf_split_by_species.xml",
+    "genomespace_importer": "genomespace/genomespace_importer.xml",
+    "gene2exon1": "filters/ucsc_gene_bed_to_exon_bed.xml",
+    "Cut1": "filters/cutWrapper.xml",
+    "Count1": "filters/uniq.xml",
+    "MAF_Limit_To_Species1": "maf/maf_limit_to_species.xml",
+    "ucsc_table_direct_test1": "data_source/ucsc_tablebrowser_test.xml",
+    "wormbase": "data_source/wormbase.xml",
+    "maf_stats1": "maf/maf_stats.xml",
+    "zebrafishmine": "data_source/zebrafishmine.xml",
+    "Paste1": "filters/pasteWrapper.xml",
+    "Interval_Maf_Merged_Fasta2": "maf/interval_maf_to_merged_fasta.xml",
+    "modENCODEworm": "data_source/worm_modencode.xml",
+    "gtf_filter_by_attribute_values_list": "filters/gff/gtf_filter_by_attribute_values_list.xml",
+    "Summary_Statistics1": "stats/gsummary.xml",
+    "qual_stats_boxplot": "plotting/boxplot.xml",
+    "cat1": "filters/catWrapper.xml",
+    "maf_by_block_number1": "maf/maf_by_block_number.xml",
+    "Grep1": "filters/grep.xml",
+    "eupathdb": "data_source/eupathdb.xml",
+    "__RELABEL_FROM_FILE__": "${model_tools_path}/relabel_from_file.xml",
+    "__ZIP_COLLECTION__": "${model_tools_path}/zip_collection.xml",
+    "lav_to_bed1": "filters/lav_to_bed.xml",
+    "comp1": "filters/compare.xml",
+    "bed_to_bigBed": "filters/bed_to_bigbed.xml",
+    "liftOver1": "extract/liftOver_wrapper.xml",
+    "bed2gff1": "filters/bed2gff.xml",
+    "__MERGE_COLLECTION__": "${model_tools_path}/merge_collection.xml",
+    "gramenemart": "data_source/gramene_mart.xml",
+    "hbvar": "data_source/hbvar.xml",
+    "yeastmine": "data_source/yeastmine.xml",
+    "Filter1": "stats/filtering.xml",
+    "flymine": "data_source/flymine.xml",
+    "wormbase_test": "data_source/wormbase_test.xml",
+    "__FILTER_FAILED_DATASETS__": "${model_tools_path}/filter_failed_collection.xml",
+    "Extract genomic DNA 1": "extract/extract_genomic_dna.xml",
+    "Remove beginning1": "filters/remove_beginning.xml",
+    "flymine_test": "data_source/flymine_test.xml",
+    "axt_to_fasta": "filters/axt_to_fasta.xml",
+    "addValue": "filters/fixedValueColumn.xml",
+    "__TAG_FROM_FILE__": "${model_tools_path}/tag_collection_from_file.xml"
+}

planemo/galaxy/workflows.py

@@ -69,11 +69,14 @@ def import_workflow(path, admin_gi, user_gi, from_path=False):
         return workflow
 
 
-def _raw_dict(path, importer):
+def _raw_dict(path, importer=None):
     if path.endswith(".ga"):
         with open(path, "r") as f:
             workflow = json.load(f)
     else:
+        if importer is None:
+            importer = DummyImporterGalaxyInterface()
+
         workflow_directory = os.path.dirname(path)
         workflow_directory = os.path.abspath(workflow_directory)
         with open(path, "r") as f:
@@ -83,13 +86,22 @@ def _raw_dict(path, importer):
     return workflow
 
 
+def find_tool_ids(path):
+    tool_ids = []
+    workflow = _raw_dict(path)
+    for (order_index, step) in workflow["steps"].items():
+        tool_id = step.get("tool_id")
+        tool_ids.append(tool_id)
+
+    return tool_ids
+
+
 WorkflowOutput = namedtuple("WorkflowOutput", ["order_index", "output_name", "label"])
 
 
 def describe_outputs(path):
     """Return a list of :class:`WorkflowOutput` objects for target workflow."""
-    importer = DummyImporterGalaxyInterface()
-    workflow = _raw_dict(path, importer)
+    workflow = _raw_dict(path)
     outputs = []
     for (order_index, step) in workflow["steps"].items():
         step_outputs = step.get("workflow_outputs", [])

planemo/test/_check_output.py

@@ -22,6 +22,10 @@ def check_output(runnable, output_properties, test_properties, **kwds):
     job_output_files = kwds.get("job_output_files", None)
     item_label = "Output with path %s" % path
     problems = []
+    if "asserts" in test_properties:
+        # TODO: break fewer abstractions here...
+        from galaxy.tools.parser.yaml import __to_test_assert_list
+        test_properties["assert_list"] = __to_test_assert_list(test_properties["asserts"])
     try:
         verify(
             item_label,

scripts/tool_index.json

@@ -0,0 +1,27 @@
+import json
+
+with open("tool_index.json", "r") as f:
+    tool_index = json.load(f)
+
+tool_map = {}
+
+for tool in tool_index:
+    tool_id = tool["id"]
+    if "/" in tool_id:
+        continue
+    config_file = tool["config_file"]
+    path = None
+    if "lib/galaxy/tools/" in config_file:
+        _, path = config_file.split("lib/galaxy/tools/", 1)
+        path = "${model_tools_path}/%s" % path
+    elif "tools/" in config_file:
+        _, path = config_file.split("tools/", 1)
+
+    if path:
+        tool_map[tool_id] = path
+
+as_python = "DISTRO_TOOLS_ID_TO_PATH = %s\n" % json.dumps(tool_map, indent=4)
+as_python = as_python.replace(" \n", "\n")
+with open("../planemo/galaxy/distro_tools.py", "w") as f:
+    f.write("# file auto generated with scripts/tool_index_to_id_map.py\n")
+    f.write(as_python)

scripts/tool_index_to_id_map.py

@@ -0,0 +1,10 @@
+- doc: Test Galaxy finds Grouping1 tool.
+  job:
+    input1:
+      class: File
+      path: '1.bed'
+  outputs:
+    wf_output_1:
+      asserts:
+        has_line:
+          line: 'chr16	206638'

tests/data/wf4-distro-tools.gxwf-test.yml

@@ -0,0 +1,18 @@
+class: GalaxyWorkflow
+name: "Test Workflow with Distro Tool"
+inputs:
+  - id: input1
+outputs:
+  - id: wf_output_1
+    source: grouping#out_file1
+steps:
+  - tool_id: Grouping1
+    label: grouping
+    state:
+      input1:
+        $link: input1
+      groupcol: 1
+      operations:
+        - opcol: '2'  # TODO: shouldn't need to be a string, but integer doesn't work here - does outside conditional :(
+          optype: mean
+          opround: 'no'  # TODO: default shouldn't need to be specified, but does because in conditional :(

tests/data/wf4-distro-tools.gxwf.yml

@@ -29,6 +29,7 @@ def test_workflow_test_simple_yaml(self):
             test_command = self.append_profile_argument_if_needed(test_command)
             test_command += [
                 "--no_dependency_resolution",
+                "--galaxy_branch", "release_18.01",  # Much better workflow output detection than master for now (pre-release of 18.01)
                 "--extra_tools", random_lines,
                 "--extra_tools", cat,
                 test_artifact,
@@ -48,6 +49,7 @@ def test_workflow_test_simple_ga(self):
             test_command = self.append_profile_argument_if_needed(test_command)
             test_command += [
                 "--no_dependency_resolution",
+                "--galaxy_branch", "release_18.01",  # Much better workflow output detection than master for now (pre-release of 18.01)
                 "--extra_tools", cat,
                 test_artifact,
             ]
@@ -58,6 +60,28 @@ def test_workflow_test_simple_ga(self):
             #        print(o.read())
             #    raise
 
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    def test_workflow_test_distro_tool(self):
+        """Test testing a simple GA workflow with Galaxy."""
+        with self._isolate():
+            test_artifact = os.path.join(TEST_DATA_DIR, "wf4-distro-tools.gxwf.yml")
+            test_command = [
+                "--verbose",
+                "test"
+            ]
+            test_command = self.append_profile_argument_if_needed(test_command)
+            test_command += [
+                "--no_dependency_resolution",
+                "--galaxy_branch", "release_18.01",  # Much better workflow output detection than master for now (pre-release of 18.01)
+                test_artifact,
+            ]
+            # try:
+            self._check_exit_code(test_command, exit_code=0)
+            # except Exception:
+            #    with open(os.path.join(f, "tool_test_output.json"), "r") as o:
+            #        print(o.read())
+            #    raise
+
     @skip_if_environ("PLANEMO_SKIP_CWLTOOL_TESTS")
     def test_cwltool_tool_test(self):
         """Test testing a CWL tool with cwltool."""