Merge pull request #874 from mvdbeek/allow_python_version_selection

Allow selection of python version when starting managed Galaxy

Author: nsoranzo

.travis.yml

@@ -1,29 +1,35 @@
 language: python
 sudo: false
-python: 2.7
 env:
   global:
     - PLANEMO_SKIP_REDUNDANT_TESTS=1
     - PLANEMO_ENABLE_POSTGRES_TESTS=1
     - PLANEMO_SKIP_GALAXY_CWL_TESTS=1
     - PLANEMO_TEST_WORKFLOW_RUN_PROFILE=travisworkflowtests
-  matrix:
-    - TOX_ENV=py27-lint
-    - TOX_ENV=py34-lint
-    - TOX_ENV=py27-lint-readme
-    - TOX_ENV=py27-lint-docs
-    - TOX_ENV=py34-quick
-    - TOX_ENV=py27
-    - TOX_ENV=py34
-    - TOX_ENV=py27-lint-docstrings
-
-install:
-  - pip install tox coveralls
-
 matrix:
+  include:
+    - python: 2.7
+      env: TOX_ENV=py27-lint
+    - python: 2.7
+      env: TOX_ENV=py27-lint-readme
+    - python: 2.7
+      env: TOX_ENV=py27-lint-docs
+    - python: 2.7
+      env: TOX_ENV=py27-lint-docstrings
+    - python: 3.6
+      env: TOX_ENV=py36-lint
+    - python: 3.6
+      env: TOX_ENV=py36-quick
+    - python: 2.7
+      env: TOX_ENV=py27
+    - python: 3.6
+      env: TOX_ENV=py36
   allow_failures:
     - env: TOX_ENV=py27-lint-docstrings
 
+install:
+  - pip install tox coveralls
+
 script: tox -e $TOX_ENV
 
 after_success:

planemo/galaxy/config.py

@@ -13,6 +13,7 @@
 from galaxy.containers.docker_model import DockerVolume
 from galaxy.tools.deps import docker_util
 from galaxy.tools.deps.commands import argv_to_str
+from pkg_resources import parse_version
 from six import (
     add_metaclass,
     iteritems
@@ -34,6 +35,7 @@
 )
 from planemo.mulled import build_involucro_context
 from planemo.shed import tool_shed_url
+from planemo.virtualenv import DEFAULT_PYTHON_VERSION
 from .api import (
     DEFAULT_MASTER_API_KEY,
     gi,
@@ -397,6 +399,10 @@ def config_join(*args):
         )
         _ensure_directory(shed_tool_path)
         port = _get_port(kwds)
+        if parse_version(kwds.get('galaxy_python_version', DEFAULT_PYTHON_VERSION)) >= parse_version('3'):
+            # on python 3 we use gunicorn,
+            # which requires 'main' as server name
+            server_name = 'main'
         template_args = dict(
             port=port,
             host=kwds.get("host", "127.0.0.1"),
@@ -941,6 +947,14 @@ def startup_command(self, ctx, **kwds):
             run_script += " --server-name %s" % shlex_quote(self.server_name)
         server_ini = os.path.join(self.config_directory, "galaxy.ini")
         self.env["GALAXY_CONFIG_FILE"] = server_ini
+        if parse_version(kwds.get('galaxy_python_version', DEFAULT_PYTHON_VERSION)) >= parse_version('3'):
+            # We need to start under gunicorn
+            self.env['APP_WEBSERVER'] = 'gunicorn'
+            self.env['GUNICORN_CMD_ARGS'] = "--bind={host}:{port} --name={server_name}".format(
+                host=kwds['host'],
+                port=kwds['port'],
+                server_name=self.server_name,
+            )
         cd_to_galaxy_command = ['cd', self.galaxy_root]
         return shell_join(
             cd_to_galaxy_command,
@@ -1233,19 +1247,13 @@ def _galaxy_source(kwds):
 
 
 def _install_with_command(ctx, config_directory, command, env, kwds):
-    # TODO: --watchdog
-    pip_installs = []
-    if pip_installs:
-        pip_install_command = ['pip', 'install'] + pip_installs
-    else:
-        pip_install_command = ""
     setup_venv_command = setup_venv(ctx, kwds)
+    env['__PYVENV_LAUNCHER__'] = ''
     install_cmd = shell_join(
         ['cd', config_directory],
         command,
         ['cd', 'galaxy-dev'],
         setup_venv_command,
-        pip_install_command,
         setup_common_startup_args(),
         COMMAND_STARTUP_COMMAND,
     )
@@ -1255,15 +1263,20 @@ def _install_with_command(ctx, config_directory, command, env, kwds):
 def _ensure_galaxy_repository_available(ctx, kwds):
     workspace = ctx.workspace
     cwl = kwds.get("cwl", False)
-    gx_repo = os.path.join(workspace, "gx_repo")
+    galaxy_source = kwds.get('galaxy_source')
+    if galaxy_source and galaxy_source != DEFAULT_GALAXY_SOURCE:
+        sanitized_repo_name = "".join(c if c.isalnum() else '_' for c in kwds['galaxy_source']).rstrip()[:255]
+        gx_repo = os.path.join(workspace, "gx_repo_%s" % sanitized_repo_name)
+    else:
+        gx_repo = os.path.join(workspace, "gx_repo")
     if cwl:
         gx_repo += "_cwl"
     if os.path.exists(gx_repo):
         # Attempt fetch - but don't fail if not interweb, etc...
-        shell("git --git-dir %s fetch >/dev/null 2>&1" % gx_repo)
+        shell("git --git-dir %s remote update >/dev/null 2>&1" % gx_repo)
     else:
         remote_repo = _galaxy_source(kwds)
-        command = git.command_clone(ctx, remote_repo, gx_repo, bare=True)
+        command = git.command_clone(ctx, remote_repo, gx_repo, mirror=True)
         shell(command)
     return gx_repo
 

planemo/galaxy/run.py

@@ -6,7 +6,10 @@
 from six.moves import shlex_quote
 
 from planemo.io import info, shell_join
-from planemo.virtualenv import create_command
+from planemo.virtualenv import (
+    create_command,
+    DEFAULT_PYTHON_VERSION,
+)
 
 
 # Activate galaxy's virtualenv if present (needed for tests say but not for
@@ -39,7 +42,7 @@ def setup_venv(ctx, kwds):
         return ""
 
     create_template_params = {
-        'create_virtualenv': create_command("$GALAXY_VIRTUAL_ENV")
+        'create_virtualenv': create_command("$GALAXY_VIRTUAL_ENV", kwds.get('galaxy_python_version'))
     }
     return shell_join(
         locate_galaxy_virtualenv(ctx, kwds),
@@ -55,6 +58,9 @@ def locate_galaxy_virtualenv(ctx, kwds):
         workspace = ctx.workspace
         galaxy_branch = kwds.get("galaxy_branch") or "master"
         shared_venv_path = os.path.join(workspace, "gx_venv")
+        galaxy_python_version = kwds.get('galaxy_python_version', DEFAULT_PYTHON_VERSION)
+        if galaxy_python_version != DEFAULT_PYTHON_VERSION:
+            shared_venv_path = "%s_%s" % (shared_venv_path, galaxy_python_version)
         if galaxy_branch != "master":
             shared_venv_path = "%s_%s" % (shared_venv_path, galaxy_branch)
         venv_command = CACHED_VIRTUAL_ENV_COMMAND % shlex_quote(shared_venv_path)

planemo/git.py

@@ -59,18 +59,17 @@ def checkout(ctx, remote_repo, local_path, branch=None, remote="origin", from_br
         io.communicate(["git", "merge", "--ff-only", "%s/%s" % (remote, from_branch)], env=env)
 
 
-def command_clone(ctx, src, dest, bare=False, branch=None):
+def command_clone(ctx, src, dest, mirror=False, branch=None):
     """Produce a command-line string to clone a repository.
 
     Take in ``ctx`` to allow more configurability down the road.
     """
-    bare_arg = ""
-    if bare:
-        bare_arg = "--bare"
-    branch_arg = ""
+    cmd = ['git', 'clone']
+    if mirror:
+        cmd.append("--mirror")
     if branch is not None:
-        branch_arg = "--branch '%s'" % branch
-    cmd = "git clone %s %s '%s' '%s'" % (bare_arg, branch_arg, src, dest)
+        cmd.extend(["--branch", branch])
+    cmd.extend([src, dest])
     return cmd
 
 

planemo/io.py

@@ -182,6 +182,10 @@ def kill_pid_file(pid_file):
 
     pid = int(open(pid_file, "r").read())
     kill_posix(pid)
+    try:
+        os.unlink(pid_file)
+    except Exception:
+        pass
 
 
 def kill_posix(pid):
@@ -323,7 +327,7 @@ def path_startswith(p):
             norm_p = norm(p)
             norm_exclude_path = norm(exclude_path)
             if norm_p.startswith(norm_exclude_path):
-                return norm_p[len(norm_exclude_path):len(norm_exclude_path)+1] in ['', os.sep]
+                return norm_p[len(norm_exclude_path):len(norm_exclude_path) + 1] in ['', os.sep]
             return False
         return path_startswith
 

planemo/options.py

@@ -129,6 +129,16 @@ def galaxy_email_option():
     )
 
 
+def galaxy_python_version():
+    return planemo_option(
+        '--galaxy_python_version',
+        use_global_config=True,
+        default='2.7',
+        type=click.Choice(['2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', '3.7']),
+        help="Python version to start Galaxy under",
+    )
+
+
 def galaxy_root_option():
     return planemo_option(
         "--galaxy_root",
@@ -1059,6 +1069,7 @@ def galaxy_config_options():
 def galaxy_target_options():
     return _compose(
         galaxy_root_option(),
+        galaxy_python_version(),
         galaxy_database_seed_option(),
         extra_tools_option(),
         install_galaxy_option(),
@@ -1153,32 +1164,32 @@ def training_topic_option():
 
 def training_tutorial_name_option():
     return planemo_option(
-            "--tutorial_name",
-            help="Name (directory name) of the tutorial to create or to modify"
+        "--tutorial_name",
+        help="Name (directory name) of the tutorial to create or to modify"
     )
 
 
 def training_tutorial_name_req_option():
     return planemo_option(
-            "--tutorial_name",
-            required=True,
-            help="Name (directory name) of the tutorial to modify"
+        "--tutorial_name",
+        required=True,
+        help="Name (directory name) of the tutorial to modify"
     )
 
 
 def training_datatype_option():
     return planemo_option(
-            "--datatypes",
-            type=click.Path(file_okay=True, resolve_path=True),
-            help="YAML file with the correspondance between Zenodo extension and Galaxy datatypes",
-            default="shared/datatypes.yaml"
+        "--datatypes",
+        type=click.Path(file_okay=True, resolve_path=True),
+        help="YAML file with the correspondance between Zenodo extension and Galaxy datatypes",
+        default="shared/datatypes.yaml"
     )
 
 
 def training_zenodo_option():
     return planemo_option(
-            "--zenodo_link",
-            help="Zenodo URL with the input data")
+        "--zenodo_link",
+        help="Zenodo URL with the input data")
 
 
 def training_tutorial_worflow_option():

planemo/shed2tap/base.py

@@ -380,12 +380,12 @@ def _tar_folders(filename):
             folders.add(i.name.rstrip("/"))
         else:
             folders.add(os.path.split(i.name)[0])
-    return folders
+    return list(folders)
 
 
 def _zip_folders(filename):
     archive = zipfile.ZipFile(filename, "r")
-    return set(i.filename.rstrip("/") for i in archive.infolist() if i.filename.endswith("/"))
+    return list(set(i.filename.rstrip("/") for i in archive.infolist() if i.filename.endswith("/")))
 
 
 def _common_prefix(folders):
@@ -523,7 +523,7 @@ def _commands_and_downloaded_file(url, target_filename=None, sha256sum=None):
         '    echo "Downloading %s"' % downloaded_filename,
         '    curl -L -o "$DOWNLOAD_CACHE/%s" "%s"' % (downloaded_filename, url),
         '    cp "$DOWNLOAD_CACHE/%s" "%s"' % (downloaded_filename, target_filename),
-        ]
+    ]
     if sha256sum:
         # This is inserted into the if-else for a fresh download only.
         # Note double space between checksum and filename:

planemo/training/tool_input.py

@@ -145,7 +145,7 @@ def get_formatted_repeat_desc(self):
                 # add description of parameters in the repeat
                 repeat_paramlist += templates.render(INPUT_SECTION, **{
                     'space': SPACE * (self.level),
-                    'section_label': "%s: %s" % (ind+1, self.tool_inp_desc['title'])})
+                    'section_label': "%s: %s" % (ind + 1, self.tool_inp_desc['title'])})
                 repeat_paramlist += paramlist_in_repeat
             self.level = cur_level
         self.wf_param_values = tmp_wf_param_values
@@ -218,17 +218,17 @@ def get_input_tool_name(step_id, steps):
 def get_empty_input():
     """Get the string for an empty input."""
     return templates.render(INPUT_FILE_TEMPLATE, **{
-        'space': 1*SPACE,
+        'space': 1 * SPACE,
         'icon': 'param-file',
         'input_name': 'Input file',
         'input_value': 'File'
-        })
+    })
 
 
 def get_empty_param():
     """Get the string for an empty param."""
     return templates.render(INPUT_PARAM, **{
-        'space': 1*SPACE,
+        'space': 1 * SPACE,
         'param_label': 'Parameter',
         'param_value': 'a value'
-        })
+    })

planemo/training/tutorial.py

@@ -227,17 +227,17 @@ def init_from_kwds(self, kwds):
         self.title = kwds["tutorial_title"]
         self.zenodo_link = kwds["zenodo_link"] if kwds["zenodo_link"] else ''
         self.questions = [
-                "Which biological questions are addressed by the tutorial?",
-                "Which bioinformatics techniques are important to know for this type of data?"]
+            "Which biological questions are addressed by the tutorial?",
+            "Which bioinformatics techniques are important to know for this type of data?"]
         self.objectives = [
-                "The learning objectives are the goals of the tutorial",
-                "They will be informed by your audience and will communicate to them and to yourself what you should focus on during the course",
-                "They are single sentences describing what a learner should be able to do once they have completed the tutorial",
-                "You can use Bloom's Taxonomy to write effective learning objectives"]
+            "The learning objectives are the goals of the tutorial",
+            "They will be informed by your audience and will communicate to them and to yourself what you should focus on during the course",
+            "They are single sentences describing what a learner should be able to do once they have completed the tutorial",
+            "You can use Bloom's Taxonomy to write effective learning objectives"]
         self.time = "3H"
         self.key_points = [
-                "The take-home messages",
-                "They will appear at the end of the tutorial"]
+            "The take-home messages",
+            "They will appear at the end of the tutorial"]
         self.contributors = ["contributor1", "contributor2"]
         self.init_wf_fp = kwds['workflow']
         self.init_wf_id = kwds['workflow_id']