Skip to content

Fix test summary report. #429

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jmchilton merged 1 commit into from
Mar 30, 2016
Merged

Fix test summary report. #429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions planemo/galaxy_test/actions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,18 @@ def __summarize_tests_full(
__summarize_test_case(structured_data_tests, testcase_el, **kwds)


def passed(xunit_testcase_el):
did_pass = True
for child_el in list(xunit_testcase_el):
if child_el.tag in ["failure", "error"]:
did_pass = False
return did_pass


def __summarize_test_case(structured_data, testcase_el, **kwds):
summary_style = kwds.get("summary")
test_id = test_structures.case_id(testcase_el)
passed = len(list(testcase_el)) == 0
if not passed:
if not passed(testcase_el):
state = click.style("failed", bold=True, fg='red')
else:
state = click.style("passed", bold=True, fg='green')
Expand Down
35 changes: 35 additions & 0 deletions tests/data/xunit_failure.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?><testsuite name="nosetests" tests="3" errors="0" failures="1" skip="0"><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000000" time="6.103"><system-out><![CDATA[Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master
]]></system-out></testcase><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000001" time="5.814"><failure type="functional.test_toolbox.JobOutputsError" message="Expected job to complete with exit code 3, found 2&#10;-------------------- &gt;&gt; begin captured stdout &lt;&lt; ---------------------&#10;Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch &amp;&amp; git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master&#10;&#10;--------------------- &gt;&gt; end captured stdout &lt;&lt; ----------------------&#10;-------------------- &gt;&gt; begin captured logging &lt;&lt; --------------------&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/users?key=test_key HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/users/2891970512fa2d5a/api_key HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/histories HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.tools: DEBUG: Validated and populated state for tool request (0.092 ms)&#10;galaxy.tools.actions: INFO: Handled output (79.310 ms)&#10;galaxy.tools.actions: INFO: Verified access to datasets (0.007 ms)&#10;galaxy.tools.execute: DEBUG: Tool [job_properties] created job [2] (230.530 ms)&#10;galaxy.tools.execute: DEBUG: Executed all jobs for tool request: (282.318 ms)&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;POST /api/tools HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs: DEBUG: (2) Working directory for job is: /tmp/tmpRWk8zh/job_working_directory/000/2&#10;galaxy.jobs.handler: DEBUG: (2) Dispatching to local runner&#10;galaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)&#10;galaxy.jobs.runners: DEBUG: Job [2] queued (85.526 ms)&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.jobs.handler: INFO: (2) Job dispatched&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs.command_factory: INFO: Built script [/tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh] for tool command[echo &quot;The bool is not true&quot;; echo &quot;The bool is very not true&quot; 1&gt;&amp;2; echo &quot;This is a different line of text.&quot; &gt; /tmp/tmpRWk8zh/files/000/dataset_2.dat; sh -c &quot;exit 2&quot;]&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;galaxy.tools.deps: DEBUG: Building dependency shell command for dependency 'samtools'&#10;galaxy.tools.deps: DEBUG: Find dependency samtools version None&#10;galaxy.tools.deps: DEBUG: Resolver tool_shed_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver galaxy_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver galaxy_packages returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver conda returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: DEBUG: Resolver conda returned &lt;galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10&gt; (isnull? True)&#10;galaxy.tools.deps: WARNING: Failed to resolve dependency on 'samtools', ignoring&#10;galaxy.jobs.runners: DEBUG: (2) command is: /tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh; return_code=$?; python &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/set_metadata_YnCx6h.py&quot; &quot;/tmp/tmpRWk8zh/tmp/tmpJPauCl&quot; &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/galaxy.json&quot; &quot;/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_in_HistoryDatasetAssociation_2_fIrrr0,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_kwds_HistoryDatasetAssociation_2_CHbEtk,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_out_HistoryDatasetAssociation_2_xEZEa_,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_results_HistoryDatasetAssociation_2_ENj5xT,/tmp/tmpRWk8zh/files/000/dataset_2.dat,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_override_HistoryDatasetAssociation_2_ylcOet&quot; 5242880; sh -c &quot;exit $return_code&quot;&#10;galaxy.jobs.runners.local: DEBUG: (2) executing job script: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh&#10;galaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;galaxy.jobs.runners.local: DEBUG: execution finished: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh&#10;galaxy.datatypes.metadata: DEBUG: loading metadata from file for: HistoryDatasetAssociation 2&#10;galaxy.jobs: DEBUG: job 2 ended (finish() executed in (497.461 ms))&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?full=true&amp;key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;requests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost&#10;requests.packages.urllib3.connectionpool: DEBUG: &quot;GET /api/histories/5729865256bc2525/contents/5729865256bc2525/display?raw=true&amp;key=00de93b579e942815740fffaafee83e6 HTTP/1.1&quot; 200 None&#10;base.twilltestcase: DEBUG: keepoutdir: /tmp/tmpRWk8zh/jobfiles, ofn: /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt&#10;base.twilltestcase: DEBUG: ## GALAXY_TEST_SAVE=/tmp/tmpRWk8zh/jobfiles. saved /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt&#10;--------------------- &gt;&gt; end captured logging &lt;&lt; ---------------------"><![CDATA[ File "/usr/lib/python2.7/unittest/case.py", line 331, in run
testMethod()
File "/home/john/workspace/galaxy/test/functional/test_toolbox.py", line 289, in test_tool
self.do_it( td )
File "/home/john/workspace/galaxy/test/functional/test_toolbox.py", line 73, in do_it
raise e
'Expected job to complete with exit code 3, found 2\n-------------------- >> begin captured stdout << ---------------------\nExecuting git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master\n\n--------------------- >> end captured stdout << ----------------------\n-------------------- >> begin captured logging << --------------------\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/users?key=test_key HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.web.framework.webapp: INFO: Session authenticated using Galaxy master api key\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/users/2891970512fa2d5a/api_key HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/histories HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.tools: DEBUG: Validated and populated state for tool request (0.092 ms)\ngalaxy.tools.actions: INFO: Handled output (79.310 ms)\ngalaxy.tools.actions: INFO: Verified access to datasets (0.007 ms)\ngalaxy.tools.execute: DEBUG: Tool [job_properties] created job [2] (230.530 ms)\ngalaxy.tools.execute: DEBUG: Executed all jobs for tool request: (282.318 ms)\nrequests.packages.urllib3.connectionpool: DEBUG: "POST /api/tools HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs: DEBUG: (2) Working directory for job is: /tmp/tmpRWk8zh/job_working_directory/000/2\ngalaxy.jobs.handler: DEBUG: (2) Dispatching to local runner\ngalaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)\ngalaxy.jobs.runners: DEBUG: Job [2] queued (85.526 ms)\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.jobs.handler: INFO: (2) Job dispatched\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs.command_factory: INFO: Built script [/tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh] for tool command[echo "The bool is not true"; echo "The bool is very not true" 1>&2; echo "This is a different line of text." > /tmp/tmpRWk8zh/files/000/dataset_2.dat; sh -c "exit 2"]\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\ngalaxy.tools.deps: DEBUG: Building dependency shell command for dependency \'samtools\'\ngalaxy.tools.deps: DEBUG: Find dependency samtools version None\ngalaxy.tools.deps: DEBUG: Resolver tool_shed_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver galaxy_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver galaxy_packages returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver conda returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: DEBUG: Resolver conda returned <galaxy.tools.deps.resolvers.NullDependency object at 0x7f7351f64d10> (isnull? True)\ngalaxy.tools.deps: WARNING: Failed to resolve dependency on \'samtools\', ignoring\ngalaxy.jobs.runners: DEBUG: (2) command is: /tmp/tmpRWk8zh/job_working_directory/000/2/tool_script.sh; return_code=$?; python "/tmp/tmpRWk8zh/job_working_directory/000/2/set_metadata_YnCx6h.py" "/tmp/tmpRWk8zh/tmp/tmpJPauCl" "/tmp/tmpRWk8zh/job_working_directory/000/2/galaxy.json" "/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_in_HistoryDatasetAssociation_2_fIrrr0,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_kwds_HistoryDatasetAssociation_2_CHbEtk,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_out_HistoryDatasetAssociation_2_xEZEa_,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_results_HistoryDatasetAssociation_2_ENj5xT,/tmp/tmpRWk8zh/files/000/dataset_2.dat,/tmp/tmpRWk8zh/job_working_directory/000/2/metadata_override_HistoryDatasetAssociation_2_ylcOet" 5242880; sh -c "exit $return_code"\ngalaxy.jobs.runners.local: DEBUG: (2) executing job script: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh\ngalaxy.jobs: DEBUG: (2) Persisting job destination (destination id: local)\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\ngalaxy.jobs.runners.local: DEBUG: execution finished: /tmp/tmpRWk8zh/job_working_directory/000/2/galaxy_2.sh\ngalaxy.datatypes.metadata: DEBUG: loading metadata from file for: HistoryDatasetAssociation 2\ngalaxy.jobs: DEBUG: job 2 ended (finish() executed in (497.461 ms))\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?full=true&key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/jobs/5729865256bc2525?key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nrequests.packages.urllib3.connectionpool: INFO: Starting new HTTP connection (1): localhost\nrequests.packages.urllib3.connectionpool: DEBUG: "GET /api/histories/5729865256bc2525/contents/5729865256bc2525/display?raw=true&key=00de93b579e942815740fffaafee83e6 HTTP/1.1" 200 None\nbase.twilltestcase: DEBUG: keepoutdir: /tmp/tmpRWk8zh/jobfiles, ofn: /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt\nbase.twilltestcase: DEBUG: ## GALAXY_TEST_SAVE=/tmp/tmpRWk8zh/jobfiles. saved /tmp/tmpRWk8zh/jobfiles/simple_line_alternative.txt\n--------------------- >> end captured logging << ---------------------']]></failure><system-out><![CDATA[Executing git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git fetch && git --work-tree test-data-cache/ad1daf4e1504da1acd42c088c6f49be6 --git-dir test-data-cache/ad1daf4e1504da1acd42c088c6f49be6/.git merge origin/master
]]></system-out><system-err><![CDATA[---------------------- >> begin tool stdout << -----------------------
The bool is not true

----------------------- >> end tool stdout << ------------------------

---------------------- >> begin tool stderr << -----------------------
The bool is very not true

----------------------- >> end tool stderr << ------------------------

]]></system-err></testcase><testcase classname="functional.test_toolbox.TestForTool_job_properties" name="test_tool_000002" time="8.770"><system-out><![CDATA[History with id 54f2a3a23292eb07 in error - summary of datasets in error below.
--------------------------------------
| 1 - Test Job Properties (HID - NAME)
| Dataset Blurb:
| error
| Dataset Info:
| The bool is true
| Fatal error: Exit code 127 (Failing exit code.)
| The bool is really true
| Dataset Job Standard Output:
| The bool is true
| Dataset Job Standard Error:
| Fatal error: Exit code 127 (Failing exit code.)
| The bool is really true
|
--------------------------------------
]]></system-out></testcase></testsuite>
13 changes: 13 additions & 0 deletions tests/test_galaxy_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
from .test_utils import TEST_DATA_DIR

from planemo.galaxy_test import structures
from planemo.galaxy_test.actions import passed

nose_1_3_report = os.path.join(TEST_DATA_DIR, "xunit_nose_1_3.xml")
nose_0_11_report = os.path.join(TEST_DATA_DIR, "xunit_nose_0_11.xml")

xunit_report_with_failure = os.path.join(TEST_DATA_DIR, "xunit_failure.xml")


def get_test_id_new():
_get_test_id(nose_1_3_report)
Expand All @@ -27,3 +30,13 @@ def _get_test_id(path):
expected_id = "functional.test_toolbox.TestForTool_cat.test_tool_000000"
assert test_id.id == expected_id
assert test_id.num == 0


def test_passed():
xml_tree = structures.parse_xunit_report(xunit_report_with_failure)
root = xml_tree.getroot()
good_testcase_el = structures.find_cases(root)[0]
assert passed(good_testcase_el)

bad_testcase_el = structures.find_cases(root)[1]
assert not passed(bad_testcase_el)
⚡