Improvements to tool test driver script.

- Run all tests for a tool by default.
- Gather a couple more things into output JSON needed to emulate what Planemo produces by combining JSON and XUnit produces from nose.
- Add --verbose flag, default to quiter testing.
- Add --append flag to gather multiple test runs together into a single test JSON file that can be converted to HTML, XUnit, or Markdown with Planemo (galaxyproject/planemo#799).

Author: jmchilton

lib/galaxy/tools/verify/interactor.py

@@ -622,7 +622,7 @@ def _verify_extra_files_content(extra_files, hda_id, dataset_fetcher, test_data_
         _verify_composite_datatype_file_content(filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
 
 
-def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job_data=None, test_index=0, tool_version=None):
+def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job_data=None, test_index=0, tool_version=None, quiet=False):
     tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
     tool_test_dict = tool_test_dicts[test_index]
     testdef = ToolTestDescription(tool_test_dict)
@@ -642,6 +642,7 @@ def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job
     job_output_exceptions = None
     tool_execution_exception = None
     expected_failure_occurred = False
+    begin_time = time.time()
     try:
         try:
             tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
@@ -662,7 +663,7 @@ def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job
             assert data_list or data_collection_list
 
             try:
-                job_stdio = _verify_outputs(testdef, test_history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor)
+                job_stdio = _verify_outputs(testdef, test_history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=quiet)
             except JobOutputsError as e:
                 job_stdio = e.job_stdio
                 job_output_exceptions = e.output_exceptions
@@ -671,16 +672,26 @@ def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job
                 job_output_exceptions = [e]
                 raise e
     finally:
-        job_data = {}
+        end_time = time.time()
+        job_data = {
+            "tool_id": tool_id,
+            "tool_version": tool_version,
+            "test_index": test_index,
+            "time_seconds": end_time - begin_time,
+        }
         if tool_inputs is not None:
             job_data["inputs"] = tool_inputs
         if job_stdio is not None:
             job_data["job"] = job_stdio
         if job_output_exceptions:
             job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
+            job_data["status"] = "failure"
         if tool_execution_exception:
             job_data["execution_problem"] = str(tool_execution_exception)
+            job_data["status"] = "error"
         if register_job_data is not None:
+            if "status" not in job_data:
+                job_data["status"] = "success"
             register_job_data(job_data)
 
     galaxy_interactor.delete_history(test_history)
@@ -698,7 +709,7 @@ def _handle_def_errors(testdef):
             raise Exception("Test parse failure")
 
 
-def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor):
+def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=False):
     assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
     job = jobs[0]
 
@@ -713,7 +724,7 @@ def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_
     found_exceptions = []
 
     def register_exception(e):
-        if not found_exceptions:
+        if not found_exceptions and not quiet:
             # Only print this stuff out once.
             for stream in ['stdout', 'stderr']:
                 if stream in job_stdio:

lib/galaxy/tools/verify/script.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import argparse
 import json
 import sys
 
 from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool
 
-DESCRIPTION = "Script to quickly run a tool test against a running Galaxy instance."
+DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
+ALL_TESTS = "*all_tests*"
 
 
 def main(argv=None):
@@ -20,35 +23,70 @@ def main(argv=None):
         "keep_outputs_dir": args.output,
     }
     tool_id = args.tool_id
-    test_index = int(args.test_index)
     tool_version = args.tool_version
 
     galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
+    raw_test_index = args.test_index
+    if raw_test_index == ALL_TESTS:
+        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
+        test_indices = [i for i in range(len(tool_test_dicts))]
+    else:
+        test_indices = [int(raw_test_index)]
 
     test_results = []
 
-    def _register_job_data(job_data):
-        test_results.append({
-            'id': tool_id + "-" + str(test_index),
-            'has_data': True,
-            'data': job_data,
-        })
-
-    try:
-        verify_tool(
-            tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, register_job_data=_register_job_data
-        )
-    finally:
-        report_obj = {
-            'version': '0.1',
-            'tests': test_results,
-        }
-        output_json = args.output_json
-        if output_json:
+    if args.append:
+        with open(args.output_json, "r") as f:
+            previous_results = json.load(f)
+            test_results = previous_results["tests"]
+
+    exceptions = []
+    verbose = args.verbose
+    for test_index in test_indices:
+        if tool_version:
+            tool_id_and_version = "%s/%s" % (tool_id, tool_version)
+        else:
+            tool_id_and_version = tool_id
+
+        test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
+
+        def register(job_data):
+            test_results.append({
+                'id': tool_id + "-" + str(test_index),
+                'has_data': True,
+                'data': job_data,
+            })
+
+        try:
+            verify_tool(
+                tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+                register_job_data=register, quiet=not verbose
+            )
+
+            if verbose:
+                print("%s passed" % test_identifier)
+
+        except Exception as e:
+            if verbose:
+                print("%s failed, %s" % (test_identifier, e))
+            exceptions.append(e)
+
+    report_obj = {
+        'version': '0.1',
+        'tests': test_results,
+    }
+    output_json = args.output_json
+    if output_json:
+        if args.output_json == "-":
+            assert not args.append
+            print(json.dumps(report_obj))
+        else:
             with open(args.output_json, "w") as f:
-                print(report_obj)
                 json.dump(report_obj, f)
 
+    if exceptions:
+        raise exceptions[0]
+
 
 def _arg_parser():
     parser = argparse.ArgumentParser(description=DESCRIPTION)
@@ -57,9 +95,11 @@ def _arg_parser():
     parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
     parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
     parser.add_argument('--tool-version', default=None, help='Tool Version')
-    parser.add_argument('-i', '--test-index', default=0, help='Tool Test Index (starting at 0)')
+    parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
     parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
-    parser.add_argument('-j', '--output_json', default=None, help='output metadata json')
+    parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
+    parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
+    parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
     return parser