Run data managers

HISTORY.rst

@@ -6,17 +6,21 @@ History
 .. to_doc
 
 ---------------------
-0.4.1.dev0
+0.5.0.dev0
 ---------------------
 
+* Add ``run-data-managers`` tool to trigger DM with multiple values and in order. `#30 <https://github.com/galaxyproject/ephemeris/pull/30>`_
+* The workflow install tool now supports a directory of workflows. `#27 <https://github.com/galaxyproject/ephemeris/pull/27>`_
+* enable global options in a tool yaml files, like `install_resolver_dependencies: true` `#26 <https://github.com/galaxyproject/ephemeris/pull/26>`_
+* Mention mimum required galaxy versions. `#23 <https://github.com/galaxyproject/ephemeris/pull/23>`_ (thanks to @mvdbeek)
 
 
 ---------------------
 0.4.0 (2016-09-07)
 ---------------------
 
 * Be more generic in determining a server time-out (thanks to @afgane).
-* Get tool list entrypoint and improvements (thank to @mvdbeek).
+* Get tool list entrypoint and improvements (thanks to @mvdbeek).
 * Rename ``tool_panel_section_name`` to ``tool_panel_section_label`` like
   ansible-galaxy-tools (thanks to @nturaga).
 * Add missing file ``tool_list.yaml.sample`` (thanks to @nturaga).

ephemeris/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-__version__ = '0.4.1.dev0'
+__version__ = '0.5.0.dev0'
 
 PROJECT_NAME = "ephemeris"
 PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"

ephemeris/run_data_managers.py

@@ -0,0 +1,82 @@
+##!/usr/bin/env python
+
+import argparse
+import logging as log
+import re
+import time
+try:
+    from urllib.parse import urljoin
+except ImportError:
+    from urlparse import urljoin
+
+import yaml
+from bioblend.galaxy import GalaxyInstance
+
+DEFAULT_URL = "http://localhost"
+
+
+def wait(gi, job):
+    """
+        Waits until a data_manager is finished or failed.
+        It will check the state of the created datasets every 30s.
+    """
+    while True:
+        value = job['outputs']
+        # check if the output of the running job is either in 'ok' or 'error' state
+        if gi.datasets.show_dataset(value[0]['id'])['state'] in ['ok', 'error']:
+            break
+        log.info('Data manager still running.')
+        time.sleep(30)
+
+
+def main(args):
+    url = args.galaxy or DEFAULT_URL
+    gi = GalaxyInstance(url=url, email=args.user, password=args.password)
+
+    # should test valid connection
+    log.info("List of valid histories: %s" % gi.users.get_current_user())
+
+    conf = yaml.load(open(args.config))
+    for dm in conf.get('data_managers'):
+        for item in dm.get('items', [None]):
+            dm_id = dm['id']
+            params = dm['params']
+            log.info('Running DM: %s' % dm_id)
+            inputs = dict()
+            # Iterate over all parameters, replace occurences of {{item}} with the current processing item
+            # and create the tool_inputs dict for running the data manager job
+            for param in params:
+                key, value = param.items()[0]
+                value = re.sub(r'{{\s*item\s*}}', item, value, flags=re.IGNORECASE)
+                inputs.update({key: value})
+
+            # run the DM-job
+            job = gi.tools.run_tool(history_id=None, tool_id=dm_id, tool_inputs=inputs)
+            wait(gi, job)
+            log.info('Reloading data managers table.')
+            for data_table in dm.get('data_table_reload', []):
+                # reload two times
+                for i in range(2):
+                    gi.make_get_request(urljoin(url, 'api/tool_data/%s/reload' % data_table))
+                    time.sleep(5)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Running Galaxy data managers in a defined order with defined parameters.')
+    parser.add_argument("-v", "--verbose", help="Increase output verbosity.",
+                        action="store_true")
+    parser.add_argument("--config", required=True, help="Path to the YAML config file with the list of data managers and data to install.")
+    parser.add_argument("-g", "--galaxy",
+                        help="Target Galaxy instance URL/IP address")
+    parser.add_argument("-u", "--user",
+                        help="Galaxy user name")
+    parser.add_argument("-p", "--password",
+                        help="Password for the Galaxy user")
+
+    args = parser.parse_args()
+    if args.verbose:
+        log.basicConfig(level=log.DEBUG)
+
+    log.info("Running data managers...")
+    main(args)

ephemeris/run_data_managers.yaml.sample

@@ -0,0 +1,32 @@
+# configuration for fetch and index genomes
+
+data_managers:
+    # Data manager ID
+    - id: toolshed.g2.bx.psu.edu/repos/devteam/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_fetch_genome_all_fasta_dbkey/0.0.2
+      # tool parameters, nested parameters should be specified using a pipe (|)
+      params:
+        - 'dbkey_source|dbkey': '{{ item }}'
+        - 'reference_source|reference_source_selector': 'ucsc'
+        - 'reference_source|requested_dbkey': '{{ item }}'
+      # Items refere to a list of variables you want to run this data manager. You can use them inside the param field with {{ item }}
+      # In case of genome for example you can run this DM with multiple genomes, or you could give multiple URLs.
+      items:
+        #- ce8
+        - dm3
+        #- mm9
+      # Name of the data-tables you want to reload after your DM are finished. This can be important for subsequent data managers
+      data_table_reload:
+        - all_fasta
+        - __dbkeys__
+
+    - id: toolshed.g2.bx.psu.edu/repos/devteam/data_manager_bowtie2_index_builder/bowtie2_index_builder_data_manager/2.2.6
+      params:
+        - 'all_fasta_source': '{{ item }}'
+      items:
+        #- ce8
+        - dm3
+        #- mm9
+      data_table_reload:
+        # Bowtie creates indices for Bowtie and TopHat
+        - bowtie2_indexes
+        - tophat2_indexes

setup.py

@@ -38,7 +38,7 @@ def get_var(var_name):
         get-tool-list=ephemeris.get_tool_list_from_galaxy:main
         shed-install=ephemeris.shed_install:script_main
         workflow-install=ephemeris.workflow_install:main
-        setup-data-libraries=ephemeris.setup_data_libraries:main
+        run-data-managers=ephemeris.run_data_managers:main
 '''
 PACKAGE_DATA = {
     # Be sure to update MANIFEST.in for source dist.