Add pytest, enable coverage testing, fix key issue

.gitignore

@@ -27,6 +27,8 @@ pip-log.txt
 .tox
 nosetests.xml
 htmlcov
+.pytest_cache
+coverage.xml
 
 # Translations
 *.mo

.travis.yml

@@ -8,6 +8,14 @@ matrix:
       env: TOX_ENV=py35-lint
     - python: 2.7
       env: TOX_ENV=py27-lint-readme
+    - python: 3.5
+      env: TOX_ENV=py35-pytest
+      services:
+      - docker
+    - python: 2.7
+      env: TOX_ENV=py27-pytest
+      services:
+      - docker
     - python: 2.7
       env: TOX_ENV=py27
       services:
@@ -16,22 +24,13 @@ matrix:
       env: TOX_ENV=py35
       services:
         - docker
-  global:
-    # do not load /etc/boto.cfg with Python 3 incompatible plugin
-    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
-    - BOTO_CONFIG=/doesnotexist
 
 
 install:
-    - sudo rm /etc/boto.cfg
-    # Setup git to allow git operations.
-    - git config --global user.name "Travis Test User"
-    - git config --global user.email "galaxy_test@galaxyproject.org"
-    - pip install tox coveralls
+    - pip install tox
     - python setup.py install
 
 script:
     - tox -e $TOX_ENV
 
-after_success:
-  - coveralls
+

MANIFEST.in

@@ -1 +1,2 @@
+graft src
 include *.rst LICENSE

dev-requirements.txt

@@ -1,7 +1,9 @@
 # For testing
 tox
-nose
 coverage
+pytest
+pytest-cov
+docker
 
 #Building Docs
 sphinx_rtd_theme
@@ -11,10 +13,6 @@ CommonMark>=0.5.6
 
 # Used to check readme.
 readme
-
-# For Python 2.6 tests
-unittest2
-
 # Used for code checking.
 pyflakes
 flake8

requirements.txt

@@ -1,5 +1,5 @@
 six>=1.9.0
-pyyaml
+PyYAML
 bioblend>=0.10.0
 Jinja2
 galaxy-lib>=18.5.7

setup.py

@@ -4,12 +4,9 @@
 import ast
 import os
 import re
-try:
-    from setuptools import setup
-except ImportError:
-    from distutils.core import setup
+from setuptools import setup, find_packages
 
-SOURCE_DIR = "ephemeris"
+SOURCE_DIR = "src/ephemeris"
 
 _version_re = re.compile(r'__version__\s+=\s+(.*)')
 
@@ -30,9 +27,6 @@ def get_var(var_name):
 
 TEST_DIR = 'tests'
 PROJECT_DESCRIPTION = 'Ephemeris is an opinionated library and set of scripts for managing the bootstrapping of Galaxy project plugins - tools, index data, and workflows.'
-PACKAGES = [
-    'ephemeris',
-]
 ENTRY_POINTS = '''
         [console_scripts]
         get-tool-list=ephemeris.get_tool_list_from_galaxy:main
@@ -73,10 +67,10 @@ def get_var(var_name):
     author=PROJECT_AUTHOR,
     author_email=PROJECT_EMAIL,
     url=PROJECT_URL,
-    packages=PACKAGES,
+    packages=find_packages('src'),
     entry_points=ENTRY_POINTS,
     package_data=PACKAGE_DATA,
-    package_dir=PACKAGE_DIR,
+    package_dir={'': 'src'},
     include_package_data=True,
     install_requires=requirements,
     license="AFL",

ephemeris/shed_tools.py → src/ephemeris/shed_tools.py

@@ -48,7 +48,7 @@
 from .ephemeris_log import disable_external_library_logging, setup_global_logger
 from .get_tool_list_from_galaxy import GiToToolYaml, the_same_repository, tools_for_repository
 from .shed_tools_args import parser
-from .shed_tools_methods import complete_repo_information, flatten_repo_info
+from .shed_tools_methods import complete_repo_information, flatten_repo_info, VALID_KEYS
 
 
 class InstallRepositoryManager(object):
@@ -110,6 +110,13 @@ def install_repositories(self,
         errored_repositories = []
         counter = 0
 
+        # Check repos for invalid keys
+        for repo in repositories:
+            for key in repo.keys():
+                if key not in VALID_KEYS and key != 'revisions':
+                    if log:
+                        log.warning("'{0}' not a valid key. Will be skipped during parsing".format(key))
+
         # Start by flattening the repo list per revision
         flattened_repos = flatten_repo_info(repositories)
         total_num_repositories = len(flattened_repos)

ephemeris/shed_tools_methods.py → src/ephemeris/shed_tools_methods.py

@@ -1,6 +1,19 @@
 from bioblend.toolshed import ToolShedInstance
 
 
+VALID_KEYS = [
+        "name",
+        "owner",
+        "changeset_revision",
+        "tool_panel_section_id",
+        "tool_panel_section_label",
+        "tool_shed_url",
+        "install_repository_dependencies",
+        "install_resolver_dependencies",
+        "install_tool_dependencies"
+    ]
+
+
 def complete_repo_information(tool, default_toolshed_url, require_tool_panel_info, default_install_tool_dependencies,
                               default_install_repository_dependencies,
                               default_install_resolver_dependencies, force_latest_revision):
@@ -67,21 +80,25 @@ def flatten_repo_info(repositories):
              that if an input element contained `revisions` key with multiple
              values, those will be returned as separate list items.
     """
+
     flattened_list = []
     for repo_info in repositories:
+        new_repo_info = dict()
+        for key, value in repo_info.items():
+            if key in VALID_KEYS:
+                new_repo_info[key] = value
         if 'revisions' in repo_info:
             revisions = repo_info.get('revisions', [])
-            repo_info.pop('revisions', None)  # Set default to avoid key error  when removing revisions
             if not revisions:  # Revisions are empty list or None
-                flattened_list.append(repo_info)
+                flattened_list.append(new_repo_info)
             else:
                 for revision in revisions:
                     # A new dictionary must be created, otherwise there will
                     # be aliasing of dictionaries. Which leads to multiple
                     # repos with the same revision in the end result.
-                    new_repo_info = dict(**repo_info)
-                    new_repo_info['changeset_revision'] = revision
-                    flattened_list.append(new_repo_info)
+                    new_revision_dict = dict(**new_repo_info)
+                    new_revision_dict['changeset_revision'] = revision
+                    flattened_list.append(new_revision_dict)
         else:  # Revision was not defined at all
-            flattened_list.append(repo_info)
+            flattened_list.append(new_repo_info)
     return flattened_list

ephemeris/sleep.py → src/ephemeris/sleep.py

@@ -36,37 +36,42 @@ def _parse_cli_options():
     return parser.parse_args()
 
 
-def main():
-    """
-    Main function
-    """
-    options = _parse_cli_options()
-
+def galaxy_wait(galaxy_url, timeout, verbose=False):
     count = 0
     while True:
         try:
-            result = requests.get(options.galaxy + '/api/version')
+            result = requests.get(galaxy_url + '/api/version')
             try:
                 result = result.json()
-                if options.verbose:
+                if verbose:
                     sys.stdout.write("Galaxy Version: %s\n" % result['version_major'])
                 break
             except ValueError:
-                if options.verbose:
+                if verbose:
                     sys.stdout.write("[%02d] No valid json returned... %s\n" % (count, result.__str__()))
                     sys.stdout.flush()
         except requests.exceptions.ConnectionError as e:
-            if options.verbose:
+            if verbose:
                 sys.stdout.write("[%02d] Galaxy not up yet... %s\n" % (count, str(e)[0:100]))
                 sys.stdout.flush()
         count += 1
 
         # If we cannot talk to galaxy and are over the timeout
-        if options.timeout != 0 and count > options.timeout:
+        if timeout != 0 and count > timeout:
             sys.stderr.write("Failed to contact Galaxy\n")
             sys.exit(1)
 
         time.sleep(1)
+
+
+def main():
+    """
+    Main function
+    """
+    options = _parse_cli_options()
+
+    galaxy_wait(options.galaxy, options.timeout, options.verbose)
+
     sys.exit(0)
 
 

tests/docker_for_galaxy.py

@@ -0,0 +1,48 @@
+from collections import namedtuple
+
+import docker
+import pytest
+from bioblend.galaxy import GalaxyInstance
+
+from ephemeris.sleep import galaxy_wait
+
+
+# It needs to work well with dev. Alternatively we can pin this to 'master' or another stable branch.
+# Preferably a branch that updates with each stable release
+GALAXY_IMAGE = "bgruening/galaxy-stable:dev"
+
+client = docker.from_env()
+
+GalaxyContainer = namedtuple('GalaxyContainer', ['url', 'container', 'attributes', 'gi'])
+
+
+# Class scope is chosen here so we can group tests on the same galaxy in a class.
+@pytest.fixture(scope="class")
+def start_container(**kwargs):
+    """Starts a docker container with the galaxy image. Returns a named tuple with the url, a GalaxyInstance object,
+    the container attributes, and the container itself."""
+    # We start a container from the galaxy image. We detach it. Port 80 is exposed to the host at a random port.
+    # The random port is because we need mac compatibility. On GNU/linux a better option would be not to expose it
+    # and use the internal ip address instead.
+    # But alas, the trappings of a proprietary BSD kernel compel us to do ugly workarounds.
+
+    container = client.containers.run(GALAXY_IMAGE, detach=True, ports={'80/tcp': None}, **kwargs)
+    container_id = container.attrs.get('Id')
+    print(container_id)
+
+    # This seems weird as we also can just get container.attrs but for some reason
+    # the network settings are not loaded in container.attrs. With the get request
+    # these attributes are loaded
+    container_attributes = client.containers.get(container_id).attrs
+
+    # Venturing into deep nested dictionaries.
+    exposed_port = container_attributes.get('NetworkSettings').get('Ports').get('80/tcp')[0].get('HostPort')
+
+    container_url = "http://localhost:{0}".format(exposed_port)
+    galaxy_wait(container_url,
+                timeout=60)  # We are only going to wait 60 seconds. These are tests, and we are impatient!
+    yield GalaxyContainer(url=container_url,
+                          container=container,
+                          attributes=container_attributes,
+                          gi=GalaxyInstance(container_url, key="admin"))
+    container.remove(force=True)

tests/test_shed_tools.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import logging
+
+from docker_for_galaxy import start_container
+
+from ephemeris.shed_tools import InstallRepositoryManager
+
+# This line is needed because flake things the import for start_container is not used otherwise.
+start_container_is_used = start_container
+
+
+# NOTE: For each series of tests that needs the same container, use the same class.
+# The start_container fixture has the "class" scope.
+
+class TestMiscellaneous(object):
+    """This class is for miscellaneous tests that can use the same galaxy container"""
+
+    def test_invalid_keys_in_repo_list(self, caplog, start_container):
+        container = start_container
+        irm = InstallRepositoryManager(container.gi)
+        caplog.set_level(logging.WARNING)
+        irm.install_repositories([
+            dict(name="bwa",
+                 owner="devteam",
+                 tool_panel_section_name="NGS: Alignment",
+                 sesame_ouvre_toi="Invalid key")
+        ], log=logging.getLogger())
+        assert "'sesame_ouvre_toi' not a valid key. Will be skipped during parsing" in caplog.text

tests/test_shed_tools_methods.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+from ephemeris.shed_tools_methods import flatten_repo_info
+
+
+def test_flatten_repo_info():
+    test_repositories = [
+        dict(name="bwa",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment",
+             revisions=["1", "2"]),
+        dict(name="bowtie2",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment",
+             changeset_revisions=["3", "4"])
+    ]
+    flattened_repos = flatten_repo_info(test_repositories)
+    assert (flattened_repos == [
+        dict(name="bwa",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment",
+             changeset_revision="1"),
+        dict(name="bwa",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment",
+             changeset_revision="2"),
+        dict(name="bowtie2",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment")
+    ])
+
+
+def test_flatten_repo_info_invalid_key():
+    test_repositories = [
+        dict(name="bwa",
+             owner="devteam",
+             tool_panel_section_label="NGS: Alignment",
+             tool_shed_url="toolshed.g2.bx.psu.edu",
+             sesame_ouvre_toi="This is an invalid key")
+    ]
+    flattened_repos = flatten_repo_info(test_repositories)
+
+    assert "sesame_ouvre_toi" not in flattened_repos[0].keys()
+    assert "tool_shed_url" in flattened_repos[0].keys()