Refactor tool testing.

- Introduce a new test command in addition to existing --test flag for install and update, try to sort out what options belong where.
- Some basic reporting for passing and failing tests.
- Have test outcomes affect the exit code (1 for install failures, 2 for installs work but tests fail).
- General refactoring of test methods into smaller pieces.

Author: jmchilton

ephemeris/shed_tools.py

@@ -68,6 +68,8 @@
 INSTALL_TOOL_DEPENDENCIES = False
 INSTALL_REPOSITORY_DEPENDENCIES = True
 INSTALL_RESOLVER_DEPENDENCIES = True
+EXIT_CODE_INSTALL_ERRORS = 1
+EXIT_CODE_TOOL_TEST_ERRORS = 2
 
 
 def _ensure_log_configured(name):
@@ -278,7 +280,7 @@ def _parser():
     # update_tools in the name space and shed-tool update will not return all the install
     # variables.
     parser.set_defaults(
-        update_tools=False,
+        action="install",
         tool_list_file=None,
         tool_yaml=None,
         owner=None,
@@ -300,22 +302,51 @@ def _parser():
              "Use shed-tools install --help for more information",
         parents=[common_arguments],
         )
-    install_command_parser.add_argument(
-        "-t", "--toolsfile",
-        dest="tool_list_file",
-        help="Tools file to use (see tool_list.yaml.sample)",)
-    install_command_parser.add_argument(
-        "-y", "--yaml_tool",
-        dest="tool_yaml",
-        help="Install tool represented by yaml string",)
-    install_command_parser.add_argument(
-        "--name",
-        help="The name of the tool to install (only applicable "
-             "if the tools file is not provided).")
-    install_command_parser.add_argument(
-        "--owner",
-        help="The owner of the tool to install (only applicable "
-             "if the tools file is not provided).")
+    update_command_parser = subparsers.add_parser(
+        "update",
+        help="This updates all tools in Galaxy to the latest revision. "
+             "Use shed-tools update --help for more information",
+        parents=[common_arguments])
+
+    test_command_parser = subparsers.add_parser(
+        "test",
+        help="This tests the supplied list of tools in Galaxy. "
+             "Use shed-tools test --help for more information",
+    )
+
+    for command_parser in [install_command_parser, test_command_parser]:
+        command_parser.add_argument(
+            "-t", "--toolsfile",
+            dest="tool_list_file",
+            help="Tools file to use (see tool_list.yaml.sample)",)
+        command_parser.add_argument(
+            "-y", "--yaml_tool",
+            dest="tool_yaml",
+            help="Install tool represented by yaml string",)
+        command_parser.add_argument(
+            "--name",
+            help="The name of the tool to install (only applicable "
+                 "if the tools file is not provided).")
+        command_parser.add_argument(
+            "--owner",
+            help="The owner of the tool to install (only applicable "
+                 "if the tools file is not provided).")
+        command_parser.add_argument(
+            "--revisions",
+            default=None,
+            nargs='*',
+            dest="revisions",
+            help="The revisions of the tool repository that will be installed. "
+                 "All revisions must be specified after this flag by a space."
+                 "Example: --revisions 0a5c7992b1ac f048033da666"
+                 "(Only applicable if the tools file is not provided).")
+        command_parser.add_argument(
+            "--toolshed",
+            dest="tool_shed_url",
+            help="The Tool Shed URL where to install the tool from. "
+                 "This is applicable only if the tool info is "
+                 "provided as an option vs. in the tools file.")
+
     install_command_parser.add_argument(
         "--section",
         dest="tool_panel_section_id",
@@ -329,21 +360,6 @@ def _parser():
         help="Galaxy tool panel section label where tool will be installed "
              "(if the section does not exist, it will be created; "
              "only applicable if the tools file is not provided).")
-    install_command_parser.add_argument(
-        "--revisions",
-        default=None,
-        nargs='*',
-        dest="revisions",
-        help="The revisions of the tool repository that will be installed. "
-             "All revisions must be specified after this flag by a space."
-             "Example: --revisions 0a5c7992b1ac f048033da666"
-             "(Only applicable if the tools file is not provided).")
-    install_command_parser.add_argument(
-        "--toolshed",
-        dest="tool_shed_url",
-        help="The Tool Shed URL where to install the tool from. "
-             "This is applicable only if the tool info is "
-             "provided as an option vs. in the tools file.")
     install_command_parser.add_argument(
         "--skip_install_tool_dependencies",
         action="store_true",
@@ -364,34 +380,40 @@ def _parser():
         dest="force_latest_revision",
         help="Will override the revisions in the tools file and always install the latest revision.")
 
-    update_command_parser = subparsers.add_parser(
-        "update",
-        help="This updates all tools in Galaxy to the latest revision."
-             "Use shed-tools update --help for more information",
-        parents=[common_arguments])
-
     for command_parser in [update_command_parser, install_command_parser]:
         command_parser.add_argument(
             "--test",
             action="store_true",
             dest="test",
             help="Run tool tests on install tools, requires Galaxy 18.05 or newer."
         )
-        command_parser.add_argument(
-            "--test_json",
-            dest="test_json",
-            help="If --test is specified, record tool test output to specified file. "
-                 "This file can be turned into nice reports with ``planemo test_reports <output.json>``."
-        )
         command_parser.add_argument(
             "--test_existing",
             action="store_true",
-            help="If --test is specified, also run tool tests on repositories already installed "
+            help="If testing tools during install, also run tool tests on repositories already installed "
                  "(i.e. skipped repositories)."
         )
+        command_parser.add_argument(
+            "--test_json",
+            dest="test_json",
+            help="If testing tools, record tool test output to specified file. "
+                 "This file can be turned into reports with ``planemo test_reports <output.json>``."
+        )
+
+    # Same test_json as above but language modified for test instead of install/update.
+    test_command_parser.add_argument(
+        "--test_json",
+        dest="test_json",
+        help="Record tool test output to specified file. "
+             "This file can be turned into reports with ``planemo test_reports <output.json>``."
+    )
 
     update_command_parser.set_defaults(
-        update_tools=True,
+        action="update",
+    )
+
+    test_command_parser.set_defaults(
+        action="test"
     )
 
     return parser
@@ -517,7 +539,7 @@ def get_install_repository_manager(options):
             'install_tool_dependencies', INSTALL_TOOL_DEPENDENCIES)
     elif options.tool_yaml:
         repositories = [yaml.safe_load(options.tool_yaml)]
-    elif options.update_tools:
+    elif options.action == "update":
         get_repository_list = GiToToolYaml(
             gi=gi,
             skip_tool_panel_section_name=False,
@@ -546,7 +568,7 @@ def get_install_repository_manager(options):
 
     install_resolver_dependencies = options.install_resolver_dependencies or install_resolver_dependencies
 
-    force_latest_revision = options.force_latest_revision or options.update_tools
+    force_latest_revision = options.force_latest_revision or options.action == "update"
 
     return InstallToolManager(repositories=repositories,
                               gi=gi,
@@ -587,10 +609,11 @@ def __init__(self,
         self.test = test
         self.test_existing = test_existing
         self.test_json = test_json
+        self.tests_passed = []
         self.test_exceptions = []
 
     def install_repositories(self):
-        """
+        """Attempt to ensure each repository in ``self.repositories`` is installed.
         """
         installation_start = dt.datetime.now()
         installed_repositories_list = installed_repository_revisions(self.gi)  # installed tools list
@@ -696,52 +719,81 @@ def install_repositories(self):
         )
         log.info("All repositories have been installed.")
         if self.test:
-            installed_tools = []
-            target_repos = self.installed_repositories
+            target_repositories = self.installed_repositories
             if self.test_existing:
-                target_repos += self.skipped_repositories
-            for installed_repository in target_repos:
-                repo_tools = tools_for_repository(self.gi, installed_repository)
-                installed_tools.extend(repo_tools)
-
-            test_results = []
-
-            for tool in installed_tools:
-                galaxy_interactor_kwds = {
-                    "galaxy_url": re.sub('/api', '', self.gi.url),
-                    "master_api_key": self.gi.key,
-                    "api_key": None,  # TODO
-                    "keep_outputs_dir": '',
-                }
-                galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
-                tool_id = tool["id"]
-                tool_version = tool["version"]
-                tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
-                test_indices = list(range(len(tool_test_dicts)))
-                for test_index in test_indices:
-                    def register(job_data):
-                        test_results.append({
-                            'id': tool_id + "-" + str(test_index),
-                            'has_data': True,
-                            'data': job_data,
-                        })
-
-                    try:
-                        verify_tool(
-                            tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
-                            register_job_data=register, quiet=True
-                        )
-                    except Exception as e:
-                        self.test_exceptions.append(e)
+                target_repositories += self.skipped_repositories
+            self.test_repositories(target_repositories=target_repositories)
+        log.info("Total run time: {0}".format(dt.datetime.now() - installation_start))
 
-            report_obj = {
-                'version': '0.1',
-                'tests': test_results,
-            }
-            with open(self.test_json or "tool_test_output.json", "w") as f:
-                json.dump(report_obj, f)
+    def test_repositories(self, target_repositories=None):
+        """Run tool tests for each tool in supplied repositories list or ``self.repositories``.
+        """
+        tool_test_start = dt.datetime.now()
+        if target_repositories is None:
+            # Consider a variant of this that doesn't even consume a tool list YAML? target
+            # something like installed_repository_revisions(self.gi)
+            target_repositories = self.repositories
+        installed_tools = []
+        for target_repository in target_repositories:
+            repo_tools = tools_for_repository(self.gi, target_repository)
+            installed_tools.extend(repo_tools)
+
+        all_test_results = []
+
+        for tool in installed_tools:
+            tool_test_results = self._test_tool(tool)
+            all_test_results.extend(tool_test_results)
+
+        report_obj = {
+            'version': '0.1',
+            'tests': all_test_results,
+        }
+        with open(self.test_json or "tool_test_output.json", "w") as f:
+            json.dump(report_obj, f)
+        log.info("Passed tool tests ({0}): {1}".format(
+            len(self.tests_passed),
+            [t for t in self.tests_passed])
+        )
+        log.info("Failed tool tests ({0}): {1}".format(
+            len(self.test_exceptions),
+            [t[0] for t in self.errored_repositories])
+        )
+        log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start))
+
+    def _test_tool(self, tool):
+        galaxy_interactor_kwds = {
+            "galaxy_url": re.sub('/api', '', self.gi.url),
+            "master_api_key": self.gi.key,
+            "api_key": None,  # TODO
+            "keep_outputs_dir": '',
+        }
+        galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
+        tool_id = tool["id"]
+        tool_version = tool["version"]
+        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
+        test_indices = list(range(len(tool_test_dicts)))
+        tool_test_results = []
+
+        for test_index in test_indices:
+            test_id = tool_id + "-" + str(test_index)
+
+            def register(job_data):
+                tool_test_results.append({
+                    'id': test_id,
+                    'has_data': True,
+                    'data': job_data,
+                })
+
+            try:
+                verify_tool(
+                    tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+                    register_job_data=register, quiet=True
+                )
+                self.tests_passed.append(test_id)
+            except Exception as e:
+                self.test_exceptions.append((test_id, e))
 
-        log.info("Total run time: {0}".format(dt.datetime.now() - installation_start))
+        return tool_test_results
 
     def create_repository_install_payload(self, repository_info):
         """
@@ -820,23 +872,28 @@ def main():
     disable_external_library_logging()
     log = setup_global_logger(name=__name__, log_file='/tmp/galaxy_tool_install.log')
     options = _parse_cli_options()
+    install_tool_manager = None
     if options.tool_list_file or options.tool_yaml or \
             options.name and options.owner and (options.tool_panel_section_id or options.tool_panel_section_label):
-        if options.update_tools:
-            sys.exit("--update can not be used together with tools to be installed.")
+        if options.action == "update":
+            sys.exit("update command can not be used together with tools to be installed.")
         install_tool_manager = get_install_repository_manager(options)
-        install_tool_manager.install_repositories()
-        if install_tool_manager.errored_repositories:
-            sys.exit(1)
+        if options.action == "test":
+            install_tool_manager.test_repositories()
+        else:
+            install_tool_manager.install_repositories()
     elif options.update_tools:
         install_tool_manager = get_install_repository_manager(options)
         install_tool_manager.install_repositories()
-        if install_tool_manager.errored_repositories:
-            sys.exit(1)
     else:
         sys.exit("Must provide a tool list file, individual tools info , a list of data manager tasks or issue the update command. "
                  "Look at usage.")
 
+    if install_tool_manager.errored_repositories:
+        sys.exit(EXIT_CODE_INSTALL_ERRORS)
+    elif install_tool_manager.test_exceptions:
+        sys.exit(EXIT_CODE_TOOL_TEST_ERRORS)
+
 
 if __name__ == "__main__":
     main()