More generalized planemo testing enhancements.

docs/_writing_cwl_intro.rst

@@ -64,6 +64,7 @@ tool is generated.
                         --example_input 2.fastq \
                         --example_output 2.fasta \
                         --container 'dukegcb/seqtk' \
+                        --test_case \
                         --help_from_command 'seqtk seq'
 
 This command generates the following CWL YAML file.
@@ -88,9 +89,49 @@ This command generates the following CWL YAML file.
     Applying linter new_draft... CHECK
     .. INFO: Modern CWL version [cwl:draft-3]
 
-A later revision of this document ill discuss defining more
-parameters for this tool and include information on generating and
-running tests with planemo for CWL tools.
+In addition to the file actual tool file, a test file will be generated
+using the example command and provided test data. The file contents are as
+follows:
+
+.. literalinclude:: writing/seqtk_seq_tests_v3.yml
+   :language: yaml
+
+This file is a planemo-specific artifact. This file may contain 1 or more
+tests - each test is an element of the top-level list. ``tool_init`` will use
+the example command to build just one test.
+
+Each test consists of a few parts:
+
+- ``doc`` - this attribute provides a short description for the test.
+- ``job`` - this can be the path to a CWL job description or a job 
+  description embedded right in the test (``tool_init`` builds the latter). 
+- ``outputs`` - this section describes the expected output for a test. Each
+  output ID of the tool or workflow under test can appear as a key. The
+  example above just describes expected specific output file contents exactly
+  but many more expectations can be described.
+
+The tests described in this file can be run using the planemo ``test`` (or
+simply ``t``) command on the original file. By default, planemo will run tool
+tests with Galaxy but we can also specify the use of ``cwltool`` (the 
+reference implementation of CWL) which will be quicker and more robust until
+while Galaxy support for the CWL is still in development.
+
+    $ planemo test --no-container --engine cwltool seqtk_seq.cwl
+    Enable beta testing mode to test artifact that isn't a Galaxy tool.
+    All 1 test(s) executed passed.
+    seqtk_seq_0: passed
+
+We can also open up the Galaxy web inteface with this tool loaded
+using the ``serve`` (or just ``s``) command.
+
+::
+
+    $ planemo s --cwl seqtk_seq.cwl
+    ...
+    serving on http://127.0.0.1:9090
+
+Open up http://127.0.0.1:9090 in a web browser to view your new 
+tool.
 
 For more information on the Common Workflow Language check out the Draft 3
 `User Guide`_ and Specification_.

docs/_writing_test_and_serve.rst

@@ -1,7 +1,7 @@
 
 ::
 
-    $ planemo t --galaxy_root=/path/to/galaxy
+    $ planemo t
     ...
     All 1 test(s) executed passed.
     seqtk_seq[0]: passed
@@ -10,8 +10,12 @@ Now we can open Galaxy with the ``serve`` (or just ``s``).
 
 ::
 
-    $ planemo s --galaxy_root=/path/to/galaxy
+    $ planemo s
     ...
     serving on http://127.0.0.1:9090
 
 Open up http://127.0.0.1:9090 in a web browser to view your new tool.
+
+Serve and test can be passed various command line arguments such as
+``--galaxy_root`` to specify a Galaxy instance to use (by default 
+planemo will download and manage a instance just for planemo).

docs/writing/gen.sh

@@ -42,6 +42,8 @@ planemo tool_init --force \
                   --example_command 'seqtk seq -a 2.fastq > 2.fasta' \
                   --example_input 2.fastq \
                   --example_output 2.fasta \
+                  --test_case \
                   --container 'dukegcb/seqtk' \
                   --help_from_command 'seqtk seq'                  
 mv seqtk_seq.cwl seqtk_seq_v3.cwl
+mv seqtk_seq_tests.yml seqtk_seq_tests_v3.yml

docs/writing/seqtk_seq_v3_tests.yml

@@ -0,0 +1,9 @@
+#!planemo test
+- doc: test generated from example command
+  job:
+    input1:
+      class: File
+      value: test-data/2.fastq
+  outputs:
+    output1:
+      path: test-data/2.fasta

planemo/commands/cmd_tool_init.py

@@ -194,23 +194,31 @@
 def cli(ctx, **kwds):
     """Generate a tool outline from supplied arguments."""
     invalid = _validate_kwds(kwds)
+    tool_id = kwds.get("id")
     if invalid:
         ctx.exit(invalid)
     output = kwds.get("tool")
     if not output:
         extension = "cwl" if kwds.get("cwl") else "xml"
-        output = "%s.%s" % (kwds.get("id"), extension)
+        output = "%s.%s" % (tool_id, extension)
     if not io.can_write_to_path(output, **kwds):
         ctx.exit(1)
     tool_description = tool_builder.build(**kwds)
-    with open(output, "w") as f:
-        f.write(tool_description.contents)
+    io.write_file(output, tool_description.contents)
     io.info("Tool written to %s" % output)
+    test_contents = tool_description.test_contents
+    if test_contents:
+        sep = "-" if "-" in tool_id else "_"
+        tests_path = "%s%stests.yml" % (kwds.get("id"), sep)
+        if not io.can_write_to_path(tests_path, **kwds):
+            ctx.exit(1)
+        io.write_file(tests_path, test_contents)
+        io.info("Tool tests written to %s" % tests_path)
+
     macros = kwds["macros"]
     macros_file = "macros.xml"
     if macros and not os.path.exists(macros_file):
-        with open(macros_file, "w") as f:
-            f.write(tool_description.macro_contents)
+        io.write_file(macros_file, tool_description.macro_contents)
     elif macros:
         io.info(REUSING_MACROS_MESSAGE)
     if tool_description.test_files:

planemo/engine/interface.py

@@ -1,6 +1,10 @@
 """Module contianing the :class:`Engine` abstraction."""
 
 import abc
+import json
+import os
+import tempfile
+
 from planemo.runnable import (
     for_path,
     cases,
@@ -97,7 +101,24 @@ def _collect_test_results(self, test_cases):
             )
             runnable = test_case.runnable
             job_path = test_case.job_path
-            run_response = self._run(runnable, job_path)
+            tmp_path = None
+            if job_path is None:
+                job = test_case.job
+                f = tempfile.NamedTemporaryFile(
+                    dir=test_case.tests_directory,
+                    suffix=".json",
+                    prefix="plnmotmptestjob",
+                    delete=False,
+                )
+                tmp_path = f.name
+                job_path = tmp_path
+                json.dump(job, f)
+                f.close()
+            try:
+                run_response = self._run(runnable, job_path)
+            finally:
+                if tmp_path:
+                    os.remove(tmp_path)
             self._ctx.vlog(
                 "Test case [%s] resulted in run response [%s]",
                 test_case,

planemo/galaxy/config.py

@@ -927,6 +927,7 @@ def _find_tool_data_table(tool_paths, test_data_dir, **kwds):
 
 
 def _search_tool_path_for(path, target, extra_paths=[]):
+    """Check for presence of a target in different artifact directories."""
     if not os.path.isdir(path):
         tool_dir = os.path.dirname(path)
     else:

planemo/galaxy/test/actions.py

@@ -136,7 +136,7 @@ def handle_reports(ctx, structured_data, kwds):
     if structured_report_file and not os.path.exists(structured_report_file):
         try:
             with open(structured_report_file, "w") as f:
-                json.dump(structured_report_file, f)
+                json.dump(structured_data, f)
         except Exception as e:
             exceptions.append(e)
 

planemo/reports/custom.js

@@ -103,9 +103,16 @@ var renderTestResults = function(testData) {
 var TestResult = function(data) {
 	this.rawId = data["id"];
 
-	var testMethod = this.rawId.split("TestForTool_")[1];
-	var toolName = testMethod.split(".test_tool_")[0];
-	var testIndex = testMethod.split(".test_tool_")[1];
+	var idParts = this.rawId.split("TestForTool_");
+	var testMethod = idParts[idParts.length-1];
+	var splitParts;
+	if(testMethod.indexOf(".test_tool_") > -1) {
+		splitParts = testMethod.split(".test_tool_");
+	} else {
+		splitParts = rSplit(testMethod, "_", 1);
+	}
+	var toolName = splitParts[0];
+	var testIndex = splitParts[1];
 	this.toolName = toolName;
 	this.testIndex = parseInt(testIndex);
 	console.log(data);
@@ -131,6 +138,12 @@ var TestResult = function(data) {
 	this.passed = (this.status == "success");
 }
 
+// http://stackoverflow.com/questions/5202085/javascript-equivalent-of-pythons-rsplit
+function rSplit(str, sep, maxsplit) {
+    var split = str.split(sep);
+    return maxsplit ? [ split.slice(0, -maxsplit).join(sep) ].concat(split.slice(-maxsplit)) : split;
+}
+
 
 // http://stackoverflow.com/questions/19491336/get-url-parameter-jquery
 function getUrlParameter(sParam)

planemo/runnable.py

@@ -21,6 +21,7 @@
 from planemo.galaxy.workflows import describe_outputs
 from planemo.exit_codes import EXIT_CODE_UNKNOWN_FILE_TYPE, ExitCodeException
 from planemo.io import error
+from planemo.test import check_output
 
 TEST_SUFFIXES = [
     "-tests", "_tests", "-test", "_test"
@@ -122,7 +123,14 @@ def normalize_to_tests_path(path):
                 i + 1, tests_path, "job"
             )
             raise Exception(message)
-        job_path = normalize_to_tests_path(test_def["job"])
+        job_def = test_def["job"]
+        if isinstance(job_def, dict):
+            job_path = None
+            job = job_def
+        else:
+            job_path = normalize_to_tests_path(job_def)
+            job = None
+
         doc = test_def.get("doc", None)
         output_expectations = test_def.get("outputs", {})
         case = TestCase(
@@ -131,6 +139,7 @@ def normalize_to_tests_path(path):
             output_expectations=output_expectations,
             index=i,
             job_path=job_path,
+            job=job,
             doc=doc,
         )
         cases.append(case)
@@ -141,10 +150,11 @@ def normalize_to_tests_path(path):
 class TestCase(object):
     """Describe an abstract test case for a specified runnable."""
 
-    def __init__(self, runnable, tests_directory, output_expectations, job_path, index, doc):
+    def __init__(self, runnable, tests_directory, output_expectations, job_path, job, index, doc):
         """Construct TestCase object from required attributes."""
         self.runnable = runnable
         self.job_path = job_path
+        self.job = job
         self.output_expectations = output_expectations
         self.tests_directory = tests_directory
         self.index = index
@@ -207,14 +217,21 @@ def structured_test_data(self, run_response):
         job_info = run_response.job_info
         if job_info is not None:
             data_dict["job"] = job_info
-        with open(self.job_path, "r") as f:
-            data_dict["inputs"] = f.read()
+        data_dict["inputs"] = self._job
         return dict(
             id=("%s_%s" % (self._test_id, self.index)),
             has_data=True,
             data=data_dict,
         )
 
+    @property
+    def _job(self):
+        if self.job_path is not None:
+            with open(self.job_path, "r") as f:
+                return f.read()
+        else:
+            return self.job
+
     def _check_output(self, output_id, output_value, output_test):
         output_problems = []
         if not isinstance(output_test, dict):
@@ -230,12 +247,14 @@ def _check_output(self, output_id, output_value, output_test):
                 output_problems.append("No path specified for expected output file [%s]" % output_id)
                 return
 
-            properties = output_properties(output_value["path"])
-            if "checksum" in output_test:
-                expected_checksum = output_test["checksum"]
-                actual_checksum = properties["checksum"]
-                if expected_checksum != actual_checksum:
-                    output_problems.append("Expected checksum [%s] does not equal computed checksum [%s]." % (expected_checksum, actual_checksum))
+            output_problems.extend(
+                check_output(
+                    self.runnable,
+                    output_value,
+                    output_test,
+                    # TODO: needs kwds in here...
+                )
+            )
 
         return output_problems
 

planemo/test/__init__.py

@@ -1 +1,7 @@
 """Module contains code for testing runnables."""
+
+from .check_output import check_output
+
+__all__ = [
+    'check_output',
+]

planemo/test/check_output.py

@@ -0,0 +1,51 @@
+"""Check an output file from a generalize artifact test."""
+from galaxy.tools.verify import verify
+
+import os
+
+
+def check_output(runnable, output_properties, test_properties, **kwds):
+    """Use galaxy-lib to check a test output.
+
+    Return a list of strings describing the problems encountered,
+    and empty list indicates no problems were detected.
+
+    Currently this will only ever return at most one detected problem because
+    of the way galaxy-lib throws exceptions instead of returning individual
+    descriptions - but this may be enhanced in the future.
+    """
+    get_filename = _test_filename_getter(runnable)
+    path = output_properties["path"]
+    output_content = open(path, "rb").read()
+    expected_file = test_properties.get("file", None)
+    job_output_files = kwds.get("job_output_files", None)
+    item_label = "Output with path %s" % path
+    problems = []
+    try:
+        verify(
+            item_label,
+            output_content,
+            attributes=test_properties,
+            filename=expected_file,
+            get_filename=get_filename,
+            keep_outputs_dir=job_output_files,
+            verify_extra_files=None,
+        )
+    except AssertionError as e:
+        problems.append(str(e))
+
+    return problems
+
+
+def _test_filename_getter(runnable):
+
+    def get_filename(name):
+        artifact_directory = os.path.dirname(runnable.path)
+        return os.path.join(artifact_directory, name)
+
+    return get_filename
+
+
+__all__ = [
+    "check_output",
+]