Enhancements for Workflow Testing

planemo/engine/interface.py

@@ -105,6 +105,7 @@ def _collect_test_results(self, test_cases):
                     suffix=".json",
                     prefix="plnmotmptestjob",
                     delete=False,
+                    mode="w+",
                 )
                 tmp_path = f.name
                 job_path = tmp_path

planemo/galaxy/config.py

@@ -41,12 +41,16 @@
     gi,
     user_api_key,
 )
+from .distro_tools import (
+    DISTRO_TOOLS_ID_TO_PATH
+)
 from .run import (
     DOWNLOAD_GALAXY,
     setup_common_startup_args,
     setup_venv,
 )
 from .workflows import (
+    find_tool_ids,
     import_workflow,
     install_shed_repos,
 )
@@ -267,12 +271,13 @@ def config_join(*args):
         _handle_job_metrics(config_directory, kwds)
 
         shed_tool_conf = "config/shed_tool_conf.xml"
-        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+        all_tool_paths = _all_tool_paths(runnables, **kwds)
 
         tool_directories = set([])  # Things to mount...
         for tool_path in all_tool_paths:
             directory = os.path.dirname(os.path.normpath(tool_path))
-            tool_directories.add(directory)
+            if os.path.exists(directory):
+                tool_directories.add(directory)
 
         # TODO: remap these.
         tool_volumes = []
@@ -334,6 +339,7 @@ def config_join(*args):
         if export_directory is not None:
             volumes.append(docker_util.DockerVolume(export_directory, "/export"))
         yield DockerGalaxyConfig(
+            ctx,
             config_directory,
             env,
             test_data_dir,
@@ -344,6 +350,7 @@ def config_join(*args):
             docker_target_kwds=docker_target_kwds,
             volumes=volumes,
             export_directory=export_directory,
+            kwds=kwds,
         )
 
 
@@ -380,6 +387,10 @@ def config_join(*args):
             galaxy_root = config_join("galaxy-dev")
 
         server_name = "planemo%d" % random.randint(0, 100000)
+        # Once we don't have to support earlier than 18.01 - try putting these files
+        # somewhere better than with Galaxy.
+        log_file = "%s.log" % server_name
+        pid_file = "%s.pid" % server_name
         _handle_dependency_resolution(ctx, config_directory, kwds)
         _handle_job_config_file(config_directory, server_name, kwds)
         _handle_job_metrics(config_directory, kwds)
@@ -390,7 +401,7 @@ def config_join(*args):
         _ensure_directory(tool_dependency_dir)
 
         shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
-        all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+        all_tool_paths = _all_tool_paths(runnables, **kwds)
         tool_definition = _tool_conf_entry_for(all_tool_paths)
         empty_tool_conf = config_join("empty_tool_conf.xml")
 
@@ -500,6 +511,10 @@ def config_join(*args):
         env["GALAXY_TEST_LOGGING_CONFIG"] = config_join("logging.ini")
         env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
         env["GALAXY_SKIP_CLIENT_BUILD"] = "1"
+        # Following are needed in 18.01 to prevent Galaxy from changing log and pid.
+        # https://github.com/galaxyproject/planemo/issues/788
+        env["GALAXY_LOG"] = log_file
+        env["GALAXY_PID"] = pid_file
         web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
         write_file(config_join("galaxy.ini"), web_config)
         tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
@@ -513,6 +528,7 @@ def config_join(*args):
         write_file(shed_data_manager_config_file, SHED_DATA_MANAGER_CONF_TEMPLATE)
 
         yield LocalGalaxyConfig(
+            ctx,
             config_directory,
             env,
             test_data_dir,
@@ -521,9 +537,23 @@ def config_join(*args):
             master_api_key,
             runnables,
             galaxy_root,
+            kwds,
         )
 
 
+def _all_tool_paths(runnables, **kwds):
+    tool_paths = [r.path for r in runnables if r.has_tools]
+    all_tool_paths = list(tool_paths) + list(kwds.get("extra_tools", []))
+    for runnable in runnables:
+        if runnable.type.name == "galaxy_workflow":
+            tool_ids = find_tool_ids(runnable.path)
+            for tool_id in tool_ids:
+                if tool_id in DISTRO_TOOLS_ID_TO_PATH:
+                    all_tool_paths.append(DISTRO_TOOLS_ID_TO_PATH[tool_id])
+
+    return all_tool_paths
+
+
 def _shared_galaxy_properties(config_directory, kwds, for_tests):
     """Setup properties useful for local and Docker Galaxy instances.
 
@@ -685,14 +715,18 @@ class BaseGalaxyConfig(GalaxyConfig):
 
     def __init__(
         self,
+        ctx,
         config_directory,
         env,
         test_data_dir,
         port,
         server_name,
         master_api_key,
         runnables,
+        kwds,
     ):
+        self._ctx = ctx
+        self._kwds = kwds
         self.config_directory = config_directory
         self.env = env
         self.test_data_dir = test_data_dir
@@ -749,7 +783,7 @@ def install_workflows(self):
                 self._install_workflow(runnable)
 
     def _install_workflow(self, runnable):
-        install_shed_repos(runnable, self.gi)
+        install_shed_repos(runnable, self.gi, self._kwds.get("ignore_dependency_problems", False))
         # TODO: Allow serialization so this doesn't need to assume a
         # shared filesystem with Galaxy server.
         from_path = runnable.type.name == "cwl_workflow"
@@ -767,6 +801,7 @@ class DockerGalaxyConfig(BaseGalaxyConfig):
 
     def __init__(
         self,
+        ctx,
         config_directory,
         env,
         test_data_dir,
@@ -777,15 +812,18 @@ def __init__(
         docker_target_kwds,
         volumes,
         export_directory,
+        kwds,
     ):
         super(DockerGalaxyConfig, self).__init__(
+            ctx,
             config_directory,
             env,
             test_data_dir,
             port,
             server_name,
             master_api_key,
             runnables,
+            kwds,
         )
         self.docker_target_kwds = docker_target_kwds
         self.volumes = volumes
@@ -853,6 +891,7 @@ class LocalGalaxyConfig(BaseGalaxyConfig):
 
     def __init__(
         self,
+        ctx,
         config_directory,
         env,
         test_data_dir,
@@ -861,15 +900,18 @@ def __init__(
         master_api_key,
         runnables,
         galaxy_root,
+        kwds,
     ):
         super(LocalGalaxyConfig, self).__init__(
+            ctx,
             config_directory,
             env,
             test_data_dir,
             port,
             server_name,
             master_api_key,
             runnables,
+            kwds,
         )
         self.galaxy_root = galaxy_root
 

planemo/galaxy/distro_tools.py

@@ -0,0 +1,99 @@
+# file auto generated with scripts/tool_index_to_id_map.py
+DISTRO_TOOLS_ID_TO_PATH = {
+    "ucsc_table_direct1": "data_source/ucsc_tablebrowser.xml",
+    "MAF_To_Fasta1": "maf/maf_to_fasta.xml",
+    "ratmine": "data_source/ratmine.xml",
+    "mousemine": "data_source/mousemine.xml",
+    "__SORTLIST__": "${model_tools_path}/sort_collection_list.xml",
+    "__UNZIP_COLLECTION__": "${model_tools_path}/unzip_collection.xml",
+    "cbi_rice_mart": "data_source/cbi_rice_mart.xml",
+    "ucsc_table_direct_archaea1": "data_source/ucsc_tablebrowser_archaea.xml",
+    "wig_to_bigWig": "filters/wig_to_bigwig.xml",
+    "ebi_sra_main": "data_source/ebi_sra.xml",
+    "metabolicmine": "data_source/metabolicmine.xml",
+    "Extract_features1": "filters/gff/extract_GFF_Features.xml",
+    "upload1": "data_source/upload.xml",
+    "wc_gnu": "filters/wc_gnu.xml",
+    "join1": "filters/joiner.xml",
+    "random_lines1": "filters/randomlines.xml",
+    "modENCODEfly": "data_source/fly_modencode.xml",
+    "gff_filter_by_attribute": "filters/gff/gff_filter_by_attribute.xml",
+    "gtf2bedgraph": "filters/gtf2bedgraph.xml",
+    "ChangeCase": "filters/changeCase.xml",
+    "__FLATTEN__": "${model_tools_path}/flatten_collection.xml",
+    "wiggle2simple1": "filters/wiggle_to_simple.xml",
+    "GeneBed_Maf_Fasta2": "maf/genebed_maf_to_fasta.xml",
+    "trimmer": "filters/trimmer.xml",
+    "createInterval": "filters/CreateInterval.xml",
+    "gff_filter_by_feature_count": "filters/gff/gff_filter_by_feature_count.xml",
+    "Interval2Maf1": "maf/interval2maf.xml",
+    "genomespace_exporter": "genomespace/genomespace_exporter.xml",
+    "Show tail1": "filters/tailWrapper.xml",
+    "barchart_gnuplot": "plotting/bar_chart.xml",
+    "microbial_import1": "data_source/microbial_import.xml",
+    "axt_to_concat_fasta": "filters/axt_to_concat_fasta.xml",
+    "__FILTER_FROM_FILE__": "${model_tools_path}/filter_from_file.xml",
+    "Interval2Maf_pairwise1": "maf/interval2maf_pairwise.xml",
+    "Show beginning1": "filters/headWrapper.xml",
+    "axt_to_lav_1": "filters/axt_to_lav.xml",
+    "modmine": "data_source/modmine.xml",
+    "__EXPORT_HISTORY__": "${model_tools_path}/imp_exp/exp_history_to_archive.xml",
+    "MAF_Thread_For_Species1": "maf/maf_thread_for_species.xml",
+    "vcf_to_maf_customtrack1": "maf/vcf_to_maf_customtrack.xml",
+    "MAF_To_BED1": "maf/maf_to_bed.xml",
+    "__DATA_FETCH__": "${model_tools_path}/data_fetch.xml",
+    "__IMPORT_HISTORY__": "${model_tools_path}/imp_exp/imp_history_from_archive.xml",
+    "biomart": "data_source/biomart.xml",
+    "Sff_extractor": "filters/sff_extractor.xml",
+    "secure_hash_message_digest": "filters/secure_hash_message_digest.xml",
+    "MAF_Reverse_Complement_1": "maf/maf_reverse_complement.xml",
+    "mergeCols1": "filters/mergeCols.xml",
+    "gff2bed1": "filters/gff2bed.xml",
+    "Grouping1": "stats/grouping.xml",
+    "maf_limit_size1": "maf/maf_limit_size.xml",
+    "sort1": "filters/sorter.xml",
+    "Convert characters1": "filters/convert_characters.xml",
+    "MAF_To_Interval1": "maf/maf_to_interval.xml",
+    "MAF_filter": "maf/maf_filter.xml",
+    "MAF_split_blocks_by_species1": "maf/maf_split_by_species.xml",
+    "genomespace_importer": "genomespace/genomespace_importer.xml",
+    "gene2exon1": "filters/ucsc_gene_bed_to_exon_bed.xml",
+    "Cut1": "filters/cutWrapper.xml",
+    "Count1": "filters/uniq.xml",
+    "MAF_Limit_To_Species1": "maf/maf_limit_to_species.xml",
+    "ucsc_table_direct_test1": "data_source/ucsc_tablebrowser_test.xml",
+    "wormbase": "data_source/wormbase.xml",
+    "maf_stats1": "maf/maf_stats.xml",
+    "zebrafishmine": "data_source/zebrafishmine.xml",
+    "Paste1": "filters/pasteWrapper.xml",
+    "Interval_Maf_Merged_Fasta2": "maf/interval_maf_to_merged_fasta.xml",
+    "modENCODEworm": "data_source/worm_modencode.xml",
+    "gtf_filter_by_attribute_values_list": "filters/gff/gtf_filter_by_attribute_values_list.xml",
+    "Summary_Statistics1": "stats/gsummary.xml",
+    "qual_stats_boxplot": "plotting/boxplot.xml",
+    "cat1": "filters/catWrapper.xml",
+    "maf_by_block_number1": "maf/maf_by_block_number.xml",
+    "Grep1": "filters/grep.xml",
+    "eupathdb": "data_source/eupathdb.xml",
+    "__RELABEL_FROM_FILE__": "${model_tools_path}/relabel_from_file.xml",
+    "__ZIP_COLLECTION__": "${model_tools_path}/zip_collection.xml",
+    "lav_to_bed1": "filters/lav_to_bed.xml",
+    "comp1": "filters/compare.xml",
+    "bed_to_bigBed": "filters/bed_to_bigbed.xml",
+    "liftOver1": "extract/liftOver_wrapper.xml",
+    "bed2gff1": "filters/bed2gff.xml",
+    "__MERGE_COLLECTION__": "${model_tools_path}/merge_collection.xml",
+    "gramenemart": "data_source/gramene_mart.xml",
+    "hbvar": "data_source/hbvar.xml",
+    "yeastmine": "data_source/yeastmine.xml",
+    "Filter1": "stats/filtering.xml",
+    "flymine": "data_source/flymine.xml",
+    "wormbase_test": "data_source/wormbase_test.xml",
+    "__FILTER_FAILED_DATASETS__": "${model_tools_path}/filter_failed_collection.xml",
+    "Extract genomic DNA 1": "extract/extract_genomic_dna.xml",
+    "Remove beginning1": "filters/remove_beginning.xml",
+    "flymine_test": "data_source/flymine_test.xml",
+    "axt_to_fasta": "filters/axt_to_fasta.xml",
+    "addValue": "filters/fixedValueColumn.xml",
+    "__TAG_FROM_FILE__": "${model_tools_path}/tag_collection_from_file.xml"
+}

planemo/galaxy/workflows.py

@@ -11,6 +11,10 @@
 from gxformat2.interface import BioBlendImporterGalaxyInterface
 from gxformat2.interface import ImporterGalaxyInterface
 
+from planemo.io import warn
+
+FAILED_REPOSITORIES_MESSAGE = "Failed to install one or more repositories."
+
 
 def load_shed_repos(runnable):
     if runnable.type.name != "galaxy_workflow":
@@ -33,14 +37,17 @@ def load_shed_repos(runnable):
     return tools
 
 
-def install_shed_repos(runnable, admin_gi):
+def install_shed_repos(runnable, admin_gi, ignore_dependency_problems):
     tools_info = load_shed_repos(runnable)
     if tools_info:
         shed_tools._ensure_log_configured("ephemeris")
         install_tool_manager = shed_tools.InstallToolManager(tools_info, admin_gi)
         install_tool_manager.install_repositories()
         if install_tool_manager.errored_repositories:
-            raise Exception("Failed to install one or more repositories.")
+            if ignore_dependency_problems:
+                warn(FAILED_REPOSITORIES_MESSAGE)
+            else:
+                raise Exception(FAILED_REPOSITORIES_MESSAGE)
 
 
 def import_workflow(path, admin_gi, user_gi, from_path=False):
@@ -62,11 +69,14 @@ def import_workflow(path, admin_gi, user_gi, from_path=False):
         return workflow
 
 
-def _raw_dict(path, importer):
+def _raw_dict(path, importer=None):
     if path.endswith(".ga"):
         with open(path, "r") as f:
             workflow = json.load(f)
     else:
+        if importer is None:
+            importer = DummyImporterGalaxyInterface()
+
         workflow_directory = os.path.dirname(path)
         workflow_directory = os.path.abspath(workflow_directory)
         with open(path, "r") as f:
@@ -76,13 +86,22 @@ def _raw_dict(path, importer):
     return workflow
 
 
+def find_tool_ids(path):
+    tool_ids = []
+    workflow = _raw_dict(path)
+    for (order_index, step) in workflow["steps"].items():
+        tool_id = step.get("tool_id")
+        tool_ids.append(tool_id)
+
+    return tool_ids
+
+
 WorkflowOutput = namedtuple("WorkflowOutput", ["order_index", "output_name", "label"])
 
 
 def describe_outputs(path):
     """Return a list of :class:`WorkflowOutput` objects for target workflow."""
-    importer = DummyImporterGalaxyInterface()
-    workflow = _raw_dict(path, importer)
+    workflow = _raw_dict(path)
     outputs = []
     for (order_index, step) in workflow["steps"].items():
         step_outputs = step.get("workflow_outputs", [])