Skip to content

Commit bbb1085

Browse files
jmchiltonjmchilton
committed
Fix test summary report.
Valid tests producing stdout/stderr would cause the summary report to report a failure. Fixes #415.
1 parent 90a3e95 commit bbb1085

3 files changed

Lines changed: 57 additions & 2 deletions

File tree

‎planemo/galaxy_test/actions.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,11 +190,18 @@ def __summarize_tests_full(
190190
__summarize_test_case(structured_data_tests, testcase_el, **kwds)
191191

192192

193+
def passed(xunit_testcase_el):
194+
did_pass = True
195+
for child_el in list(xunit_testcase_el):
196+
if child_el.tag in ["failure", "error"]:
197+
did_pass = False
198+
return did_pass
199+
200+
193201
def __summarize_test_case(structured_data, testcase_el, **kwds):
194202
summary_style = kwds.get("summary")
195203
test_id = test_structures.case_id(testcase_el)
196-
passed = len(list(testcase_el)) == 0
197-
if not passed:
204+
if not passed(testcase_el):
198205
state = click.style("failed", bold=True, fg='red')
199206
else:
200207
state = click.style("passed", bold=True, fg='green')

‎tests/data/xunit_failure.xml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?xml version="1.0" encoding="UTF-8"?><testsuite name="nosetests" tests="3" errors="0" failures="1" skip="0"><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000000" time="6.103"><system-out><![CDATA[Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master
2+
]]></system-out></testcase><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000001" time="5.814"><failure type="functional.test_toolbox.JobOutputsError" message="Expected job to complete with exit code 3, found 2&#10;-------------------- &gt;&gt; begin captured stdout &lt;&lt; ---------------------&#10;Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch &amp;&amp; git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master&#10;&#10;--------------------- &gt;&gt; end captured stdout &lt;&lt; ----------------------&#10;-------------------- &gt;&gt; begin captured logging &lt;&lt; --------------------&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/users?key=test_key HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/users/2891970512fa2d5a/api_key HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/histories HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.tools: DEBUG: Validated and populated state for tool request (0.092 ms)&#10;galaxy.tools.actions: INFO: Handled output (79.310 ms)&#10;galaxy.tools.actions: INFO: Verified access to datasets (0.007 ms)&#10;galaxy.tools.execute: DEBUG: Tool [job_properties] created job [2] (230.530 ms)&#10;galaxy.tools.execute: DEBUG: Executed all jobs for tool request: (282.318 ms)&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/tools HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs: DEBUG: (2) Working directory for job is: /tmp/tmpRWk8zh/job_working_directory/000/2&#10;galaxy.jobs.handler: DEBUG: (2) Dispatching to local runner&#10;galaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)&#10;galaxy.jobs.runners: DEBUG: Job [2] queued (85.526 ms)&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.jobs.handler: INFO: (2) Job dispatched&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs.command_factory: INFO: Built script [/tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh] for tool command[echo &quot;The bool is not true&quot;; echo &quot;The bool is very not true&quot; 1&gt;&amp;2; echo &quot;This is a different line of text.&quot; &gt; /tmp/tmpRWk8zh/files/000/dataset_2.dat; sh -c &quot;exit 2&quot;]&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.tools.deps: DEBUG: Building dependency shell command for dependency 'samtools'&#10;galaxy.tools.deps: DEBUG: Find dependency samtools version None&#10;galaxy.tools.deps: DEBUG: Resolver tool_shed_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver galaxy_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver galaxy_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver conda returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver conda returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: WARNING: Failed to resolve dependency on 'samtools', ignoring&#10;galaxy.jobs.runners: DEBUG: (2) command is: /tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh; return_code=$?; python &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/set_metadata_YnCx6h.py&quot; &quot;/tmp/tmpRWk8zh/tmp/tmpJPauCl&quot; &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/galaxy.json&quot; &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_in_HistoryDatasetAssociation_2_fIrrr0,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_kwds_HistoryDatasetAssociation_2_CHbEtk,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_out_HistoryDatasetAssociation_2_xEZEa_,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_results_HistoryDatasetAssociation_2_ENj5xT,/tmp/tmpRWk8zh/files/000/dataset_2.dat,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_override_HistoryDatasetAssociation_2_ylcOet&quot; 5242880; sh -c &quot;exit $return_code&quot;&#10;galaxy.jobs.runners.local: DEBUG: (2) executing job script: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh&#10;galaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs.runners.local: DEBUG: execution finished: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh&#10;galaxy.datatypes.metadata: DEBUG: loading metadata from file for: HistoryDatasetAssociation 2&#10;galaxy.jobs: DEBUG: job 2 ended (finish() executed in (497.461 ms))&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?full=true&amp;key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/histories/5729865256bc2525/contents/5729865256bc2525/display?raw=true&amp;key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;base.twilltestcase: DEBUG: keepoutdir: /tmp/tmpRWk8zh/jobfiles, ofn: /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt&#10;base.twilltestcase: DEBUG: ## GALAXY_TEST_SAVE=/tmp/tmpRWk8zh/jobfiles. saved /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt&#10;--------------------- &gt;&gt; end captured logging &lt;&lt; ---------------------"><![CDATA[ File "/usr/lib/python2.7/unittest/case.py", line 331, in run
3+
testMethod()
4+
File "/home/john/workspace/galaxy/test/functional/test_toolbox.py", line 289, in test_tool
5+
self.do_it( td )
6+
File "/home/john/workspace/galaxy/test/functional/test_toolbox.py", line 73, in do_it
7+
raise e
8+
'Expected job to complete with exit code 3, found 2\n-------------------- >> begin captured stdout << ---------------------\nExecuting git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master\n\n--------------------- >> end captured stdout << ----------------------\n-------------------- >> begin captured logging << --------------------\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/users?key=test_key HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/users/2891970512fa2d5a/api_key HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/histories HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.tools: DEBUG: Validated and populated state for tool request (0.092 ms)\ngalaxy.tools.actions: INFO: Handled output (79.310 ms)\ngalaxy.tools.actions: INFO: Verified access to datasets (0.007 ms)\ngalaxy.tools.execute: DEBUG: Tool [job_properties] created job [2] (230.530 ms)\ngalaxy.tools.execute: DEBUG: Executed all jobs for tool request: (282.318 ms)\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/tools HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs: DEBUG: (2) Working directory for job is: /tmp/tmpRWk8zh/job_working_directory/000/2\ngalaxy.jobs.handler: DEBUG: (2) Dispatching to local runner\ngalaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)\ngalaxy.jobs.runners: DEBUG: Job [2] queued (85.526 ms)\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.jobs.handler: INFO: (2) Job dispatched\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs.command_factory: INFO: Built script [/tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh] for tool command[echo "The bool is not true"; echo "The bool is very not true" 1>&2; echo "This is a different line of text." > /tmp/tmpRWk8zh/files/000/dataset_2.dat; sh -c "exit 2"]\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.tools.deps: DEBUG: Building dependency shell command for dependency \'samtools\'\ngalaxy.tools.deps: DEBUG: Find dependency samtools version None\ngalaxy.tools.deps: DEBUG: Resolver tool_shed_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver galaxy_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver galaxy_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver conda returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver conda returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: WARNING: Failed to resolve dependency on \'samtools\', ignoring\ngalaxy.jobs.runners: DEBUG: (2) command is: /tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh; return_code=$?; python "/tmp/tmpRWk8zh/job_working_directory/000/2/set_metadata_YnCx6h.py" "/tmp/tmpRWk8zh/tmp/tmpJPauCl" "/tmp/tmpRWk8zh/job_working_directory/000/2/galaxy.json" "/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_in_HistoryDatasetAssociation_2_fIrrr0,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_kwds_HistoryDatasetAssociation_2_CHbEtk,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_out_HistoryDatasetAssociation_2_xEZEa_,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_results_HistoryDatasetAssociation_2_ENj5xT,/tmp/tmpRWk8zh/files/000/dataset_2.dat,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_override_HistoryDatasetAssociation_2_ylcOet" 5242880; sh -c "exit $return_code"\ngalaxy.jobs.runners.local: DEBUG: (2) executing job script: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh\ngalaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs.runners.local: DEBUG: execution finished: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh\ngalaxy.datatypes.metadata: DEBUG: loading metadata from file for: HistoryDatasetAssociation 2\ngalaxy.jobs: DEBUG: job 2 ended (finish() executed in (497.461 ms))\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?full=true&key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/histories/5729865256bc2525/contents/5729865256bc2525/display?raw=true&key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nbase.twilltestcase: DEBUG: keepoutdir: /tmp/tmpRWk8zh/jobfiles, ofn: /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt\nbase.twilltestcase: DEBUG: ## GALAXY_TEST_SAVE=/tmp/tmpRWk8zh/jobfiles. saved /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt\n--------------------- >> end captured logging << ---------------------']]></failure><system-out><![CDATA[Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master
9+
]]></system-out><system-err><![CDATA[---------------------- >> begin tool stdout << -----------------------
10+
The bool is not true
11+
12+
----------------------- >> end tool stdout << ------------------------
13+
14+
---------------------- >> begin tool stderr << -----------------------
15+
The bool is very not true
16+
17+
----------------------- >> end tool stderr << ------------------------
18+
19+
]]></system-err></testcase><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000002" time="8.770"><system-out><![CDATA[History with id 54f2a3a23292eb07 in error - summary of datasets in error below.
20+
--------------------------------------
21+
| 1 - Test Job Properties (HID - NAME)
22+
| Dataset Blurb:
23+
| error
24+
| Dataset Info:
25+
| The bool is true
26+
| Fatal error: Exit code 127 (Failing exit code.)
27+
| The bool is really true
28+
| Dataset Job Standard Output:
29+
| The bool is true
30+
| Dataset Job Standard Error:
31+
| Fatal error: Exit code 127 (Failing exit code.)
32+
| The bool is really true
33+
|
34+
--------------------------------------
35+
]]></system-out></testcase></testsuite>

‎tests/test_galaxy_test.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@
55
from .test_utils import TEST_DATA_DIR
66

77
from planemo.galaxy_test import structures
8+
from planemo.galaxy_test.actions import passed
89

910
nose_1_3_report = os.path.join(TEST_DATA_DIR, "xunit_nose_1_3.xml")
1011
nose_0_11_report = os.path.join(TEST_DATA_DIR, "xunit_nose_0_11.xml")
1112

13+
xunit_report_with_failure = os.path.join(TEST_DATA_DIR, "xunit_failure.xml")
14+
1215

1316
def get_test_id_new():
1417
_get_test_id(nose_1_3_report)
@@ -27,3 +30,13 @@ def _get_test_id(path):
2730
expected_id = "functional.test_toolbox.TestForTool_cat.test_tool_000000"
2831
assert test_id.id == expected_id
2932
assert test_id.num == 0
33+
34+
35+
def test_passed():
36+
xml_tree = structures.parse_xunit_report(xunit_report_with_failure)
37+
root = xml_tree.getroot()
38+
good_testcase_el = structures.find_cases(root)[0]
39+
assert passed(good_testcase_el)
40+
41+
bad_testcase_el = structures.find_cases(root)[1]
42+
assert not passed(bad_testcase_el)

0 commit comments

Comments
 (0)
⚡