Merge pull request #849 from jmchilton/galaxy_versions

Better support for different versions of Galaxy.

Author: jmchilton

planemo/cli.py

@@ -2,10 +2,12 @@
 import functools
 import logging.config
 import os
+import shutil
 import sys
 import traceback
 
 import click
+from six.moves.urllib.request import urlopen
 
 from planemo import __version__
 from planemo.exit_codes import ExitCodeException
@@ -112,6 +114,21 @@ def exit(self, exit_code):
         self.vlog("Exiting planemo with exit code [%d]" % exit_code)
         raise ExitCodeException(exit_code)
 
+    def cache_download(self, url, destination):
+        cache = os.path.join(self.workspace, "cache")
+        if not os.path.exists(cache):
+            os.makedirs(cache)
+        filename = os.path.basename(url)
+        cache_destination = os.path.join(cache, filename)
+        if not os.path.exists(cache_destination):
+            content = urlopen(url).read()
+            if len(content) == 0:
+                raise Exception("Failed to download [%s]." % url)
+            with open(cache_destination, "wb") as f:
+                f.write(content)
+
+        shutil.copy(cache_destination, destination)
+
 
 pass_context = click.make_pass_decorator(Context, ensure=True)
 cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__),

planemo/galaxy/config.py

@@ -13,13 +13,10 @@
 from galaxy.containers.docker_model import DockerVolume
 from galaxy.tools.deps import docker_util
 from galaxy.tools.deps.commands import argv_to_str
-from galaxy.util import unicodify
 from six import (
     add_metaclass,
     iteritems
 )
-from six.moves.urllib.request import urlopen
-from six.moves.urllib.request import urlretrieve
 
 from planemo import git
 from planemo.config import OptionSource
@@ -189,7 +186,15 @@ class = StreamHandler
 DOWNLOADS_URL = ("https://raw.githubusercontent.com/"
                  "jmchilton/galaxy-downloads/master/")
 DOWNLOADABLE_MIGRATION_VERSIONS = [141, 127, 120, 117]
-LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
+MIGRATION_PER_VERSION = {
+    "master": 141,
+    "18.05": 141,
+    "18.01": 140,
+    "17.09": 135,
+    "17.05": 134,
+    "17.01": 133,
+}
+OLDEST_SUPPORTED_VERSION = 127
 
 DATABASE_LOCATION_TEMPLATE = "sqlite:///%s?isolation_level=IMMEDIATE"
 
@@ -345,14 +350,12 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
         def config_join(*args):
             return os.path.join(config_directory, *args)
 
-        latest_galaxy = False
         install_env = {
             'GALAXY_SKIP_CLIENT_BUILD': '1',
         }
         if install_galaxy:
             _build_eggs_cache(ctx, install_env, kwds)
             _install_galaxy(ctx, config_directory, install_env, kwds)
-            latest_galaxy = True
             galaxy_root = config_join("galaxy-dev")
 
         server_name = "planemo%d" % random.randint(0, 100000)
@@ -388,9 +391,9 @@ def config_join(*args):
         master_api_key = _get_master_api_key(kwds)
         dependency_dir = os.path.join(config_directory, "deps")
         preseeded_database = attempt_database_preseed(
+            ctx,
             galaxy_root,
             database_location,
-            latest_galaxy=latest_galaxy,
             **kwds
         )
         _ensure_directory(shed_tool_path)
@@ -982,7 +985,7 @@ def _database_connection(database_location, **kwds):
 
 
 def attempt_database_preseed(
-    effective_galaxy_root, database_location, latest_galaxy=False, **kwds
+    ctx, effective_galaxy_root, database_location, **kwds
 ):
     """If database location is unset, attempt to seed the database."""
     if os.path.exists(database_location):
@@ -995,12 +998,14 @@ def attempt_database_preseed(
 
     preseeded_database = True
     galaxy_sqlite_database = kwds.get("galaxy_database_seed", None)
+    galaxy_branch = kwds.get("galaxy_branch", None)
     try:
         _download_database_template(
+            ctx,
             effective_galaxy_root,
             database_location,
-            latest=latest_galaxy,
             galaxy_sqlite_database=galaxy_sqlite_database,
+            galaxy_branch=galaxy_branch
         )
     except Exception as e:
         print(e)
@@ -1010,40 +1015,43 @@ def attempt_database_preseed(
 
 
 def _download_database_template(
+    ctx,
     galaxy_root,
     database_location,
-    latest=False,
-    galaxy_sqlite_database=None
+    galaxy_sqlite_database=None,
+    galaxy_branch=None,
 ):
+
     if galaxy_sqlite_database is not None:
         shutil.copyfile(galaxy_sqlite_database, database_location)
         return True
 
-    if latest or not galaxy_root:
-        symlink_target = unicodify(urlopen(LATEST_URL).read())
-        template_url = DOWNLOADS_URL + symlink_target
-        urlretrieve(template_url, database_location)
-        return True
-
-    newest_migration = _newest_migration_version(galaxy_root)
     download_migration = None
+    newest_migration = _newest_migration_version(galaxy_root, galaxy_branch)
     for migration in DOWNLOADABLE_MIGRATION_VERSIONS:
-        if newest_migration > migration:
+        if newest_migration >= migration:
             download_migration = migration
             break
 
     if download_migration:
         download_name = "db_gx_rev_0%d.sqlite" % download_migration
         download_url = DOWNLOADS_URL + download_name
-        urlretrieve(download_url, database_location)
+        ctx.cache_download(download_url, database_location)
         return True
     else:
         return False
 
 
-def _newest_migration_version(galaxy_root):
-    versions = os.path.join(galaxy_root, "lib/galaxy/model/migrate/versions")
-    version = max(map(_file_name_to_migration_version, os.listdir(versions)))
+def _newest_migration_version(galaxy_root, galaxy_branch):
+    versions_dir = galaxy_root and os.path.join(galaxy_root, "lib/galaxy/model/migrate/versions")
+    if versions_dir and os.path.exists(versions_dir):
+        version = max(map(_file_name_to_migration_version, os.listdir(versions_dir)))
+    elif galaxy_branch:
+        for branch in MIGRATION_PER_VERSION:
+            if branch in galaxy_branch:
+                version = MIGRATION_PER_VERSION[branch]
+    else:
+        version = OLDEST_SUPPORTED_VERSION
     return version
 
 

planemo/galaxy/profiles.py

@@ -59,22 +59,22 @@ def create_profile(ctx, profile_name, **kwds):
 
     os.makedirs(profile_directory)
     create_for_engine = _create_profile_docker if engine_type == "docker_galaxy" else _create_profile_local
-    stored_profile_options = create_for_engine(profile_directory, profile_name, kwds)
+    stored_profile_options = create_for_engine(ctx, profile_directory, profile_name, kwds)
 
     profile_options_path = _stored_profile_options_path(profile_directory)
     with open(profile_options_path, "w") as f:
         json.dump(stored_profile_options, f)
 
 
-def _create_profile_docker(profile_directory, profile_name, kwds):
+def _create_profile_docker(ctx, profile_directory, profile_name, kwds):
     export_directory = os.path.join(profile_directory, "export")
     os.makedirs(export_directory)
     return {
         "engine": "docker_galaxy",
     }
 
 
-def _create_profile_local(profile_directory, profile_name, kwds):
+def _create_profile_local(ctx, profile_directory, profile_name, kwds):
     database_type = kwds.get("database_type", "auto")
     if database_type == "auto":
         if which("psql"):
@@ -93,7 +93,7 @@ def _create_profile_local(profile_directory, profile_name, kwds):
         database_connection = database_source.sqlalchemy_url(database_identifier)
     else:
         database_location = os.path.join(profile_directory, "galaxy.sqlite")
-        attempt_database_preseed(None, database_location, **kwds)
+        attempt_database_preseed(ctx, None, database_location, **kwds)
         database_connection = DATABASE_LOCATION_TEMPLATE % database_location
 
     return {

planemo/galaxy/run.py

@@ -53,7 +53,10 @@ def setup_venv(ctx, kwds):
 def locate_galaxy_virtualenv(ctx, kwds):
     if not kwds.get("no_cache_galaxy", False):
         workspace = ctx.workspace
+        galaxy_branch = kwds.get("galaxy_branch", "master")
         shared_venv_path = os.path.join(workspace, "gx_venv")
+        if galaxy_branch != "master":
+            shared_venv_path = "%s_%s" % (shared_venv_path, galaxy_branch)
         venv_command = CACHED_VIRTUAL_ENV_COMMAND % shared_venv_path
     else:
         venv_command = UNCACHED_VIRTUAL_ENV_COMMAND