Glib2 travis issue [WIP] (#83)

* hoping to help travis not use /lib64

* since issue is only linux turning off osx for now
also uploading on all successes

* while testing turning off other numpys

* ok trying to run a docker now

* another try at docker

* no if test while testing

* make sure conda_upload is executable, added bash header to it, full path for docker

* -c is not needed

* installing missing packages on centos6

* more verbose docker

* gfortra needed

* quiet because output seems too long

* need to print env to see what is really passed

* trying to use our custom docker image with everything pre-loaded

* seems like networks hangs so putting a retry

* for the purpose of testing docker turing off actuall travis bits

* get latest docker image with conda-forge pkg pre-downloaded

build numpy 1.11, 1.10 , 1.9

* trying to matrix oses

* need to put os before matrix?

* dropping matrix can't figure it out

* trying to add circle ci

* needs a checkout, copied from conda-forge

* reorg a bit

* ok actually running this thing now on circle-ci

* anonymous CDAT

* token removed it was in clear, wrong location for env variable

* putting mac back in for travis

Author: doutriaux1

.travis.yml

@@ -1,7 +1,11 @@
 os:
     - osx
     - linux
-sudo: false
+
+sudo: required
+services:
+    - docker
+
 before_install:
 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi
 - if [ "$TRAVIS_OS_NAME" = "osx" ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
@@ -14,7 +18,7 @@ script:
 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -c conda-forge -c uvcdat uvcdat-nox pyopenssl; fi
 - if [ "$TRAVIS_OS_NAME" = "osx" ]; then conda install -c conda-forge -c uvcdat uvcdat pyopenssl; fi
 - export UVCDAT_ANONYMOUS_LOG=False
-- vcs_download_sample_data
+- travis_retry vcs_download_sample_data
 - echo $TRAVIS_BRANCH
 - export TRAVIS_PR_BRANCH=$TRAVIS_BRANCH
 - echo $TRAVIS_EVENT_TYPE
@@ -25,4 +29,5 @@ env:
   global:
     secure: Z06idl4BP01QMhrocmfuYV4CcbLmb5ZJZoXXymDr5sLIIfqdMiSL/CfDCRkIwl44vKpTN5R9AAhQV6iNMdU0NOQQrftHJyT5YV4y966FuAA1v9ev7y/cgExJFv7M/4E37WAN6YIcO7E42KZReeNKOAodorpqoT87MW6FAImZbrM=
 after_success:
-  - if [ $TRAVIS_BRANCH == "master" -a $TRAVIS_PULL_REQUEST == "false" ]; then conda install conda-build && conda install anaconda-client && bash conda_upload.sh ; fi
+    - if [ $TRAVIS_BRANCH == "master" -a $TRAVIS_PULL_REQUEST == "false" -a "$TRAVIS_OS_NAME" = "osx"]; then conda install conda-build && conda install anaconda-client && bash scripts/conda_upload.sh ; fi
+    - if [ $TRAVIS_BRANCH == "master" -a $TRAVIS_PULL_REQUEST == "false" -a "$TRAVIS_OS_NAME" = "linux"]; docker run -v `pwd`:/travis_home -e CONDA_UPLOAD_TOKEN=${CONDA_UPLOAD_TOKEN} -a STDOUT cdat/conda:conda-forge-cdms2 /travis_home/scripts/conda_upload.sh ; fi

circle.yml

@@ -0,0 +1,24 @@
+general:
+  branches:
+    ignore:
+      # We only want to build pull requests for testing. If something is merged,
+      # then we are prepping for release an there is no need to build it again.
+      - master
+
+checkout:
+  post:
+    - ./scripts/checkout_merge_commit.sh
+
+machine:
+  services:
+    - docker
+
+dependencies:
+  # Note, we used to use the naive caching of docker images, but found that it was quicker
+  # just to pull each time. #rollondockercaching
+  override:
+    - docker pull cdat/conda:conda-forge-cdms2
+
+test:
+  override:
+      - docker run -it -v `pwd`:/git_repo -a STDOUT -a STDERR -P cdat/conda:conda-forge-cdms2 /git_repo/scripts/circle.sh

scripts/checkout_merge_commit.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+
+# Add `master` branch to the update list.
+# Otherwise CircleCI will give us a cached one.
+FETCH_REFS="+master:master"
+
+# Update PR refs for testing.
+if [[ -n "${CIRCLE_PR_NUMBER}" ]]
+then
+    FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/head:pr/${CIRCLE_PR_NUMBER}/head"
+    FETCH_REFS="${FETCH_REFS} +refs/pull/${CIRCLE_PR_NUMBER}/merge:pr/${CIRCLE_PR_NUMBER}/merge"
+fi
+
+# Retrieve the refs.
+git fetch -u origin ${FETCH_REFS}
+
+# Checkout the PR merge ref.
+if [[ -n "${CIRCLE_PR_NUMBER}" ]]
+then
+    git checkout -qf "pr/${CIRCLE_PR_NUMBER}/merge"
+fi
+
+# Check for merge conflicts.
+if [[ -n "${CIRCLE_PR_NUMBER}" ]]
+then
+    git branch --merged | grep master > /dev/null
+    git branch --merged | grep "pr/${CIRCLE_PR_NUMBER}/head" > /dev/null
+fi

scripts/circle.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+PKG_NAME=cdms2
+USER=uvcdat
+export PATH=${HOME}/miniconda2/bin:${PATH}
+echo "Trying to upload conda"
+conda update -y -q conda
+conda install -c conda-forge -c uvcdat uvcdat-nox pyopenssl
+export UVCDAT_ANONYMOUS_LOG=False
+vcs_download_sample_data
+#mkdir ${HOME}/conda-bld
+conda config --set anaconda_upload no
+#export CONDA_BLD_PATH=${HOME}/conda-bld
+#export VERSION=`date +%Y.%m.%d`
+echo "Cloning recipes"
+cd /git_repo
+ls -l
+python setup.py install
+python run_tests.py
+#git clone git://github.com/UV-CDAT/conda-recipes
+#cd conda-recipes
+# uvcdat creates issues for build -c uvcdat confises package and channel
+#rm -rf uvcdat
+#python ./prep_for_build.py -v `date +%Y.%m.%d`
+#echo "Building now"
+#conda build -c conda-forge -c uvcdat --numpy=1.11 cdms2
+#echo "Uploading"
+#anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-`date +%Y.%m.%d`-np111py27_0.tar.bz2 --force
+#conda build cdms2 -c conda-forge -c uvcdat --numpy=1.10
+#anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-`date +%Y.%m.%d`-np110py27_0.tar.bz2 --force
+#conda build cdms2 -c conda-forge -c uvcdat --numpy=1.9
+#anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-`date +%Y.%m.%d`-np19py27_0.tar.bz2 --force
+

conda_upload.sh scripts/conda_upload.shconda_upload.sh renamed to scripts/conda_upload.sh

@@ -1,19 +1,30 @@
+#!/usr/bin/env bash
 PKG_NAME=cdms2
 USER=uvcdat
-if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+echo "Trying to upload conda"
+if [ `uname` == "Linux" ]; then
     OS=linux-64
+    echo "Linux OS"
+    export PATH="$HOME/miniconda2/bin:$PATH"
+    conda update -y -q conda
 else
+    echo "Mac OS"
     OS=osx-64
 fi
 
 mkdir ~/conda-bld
 conda config --set anaconda_upload no
-export CONDA_BLD_PATH=~/conda-bld
+export CONDA_BLD_PATH=${HOME}/conda-bld
 export VERSION=`date +%Y.%m.%d`
+echo "Cloning recipes"
 git clone git://github.com/UV-CDAT/conda-recipes
 cd conda-recipes
+# uvcdat creates issues for build -c uvcdat confises package and channel
+rm -rf uvcdat
 python ./prep_for_build.py -v `date +%Y.%m.%d`
-conda build cdms2 -c conda-forge -c uvcdat --numpy=1.11
+echo "Building now"
+conda build -c conda-forge -c uvcdat --numpy=1.11 cdms2
+echo "Uploading"
 anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-`date +%Y.%m.%d`-np111py27_0.tar.bz2 --force
 conda build cdms2 -c conda-forge -c uvcdat --numpy=1.10
 anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-`date +%Y.%m.%d`-np110py27_0.tar.bz2 --force

setup.py

@@ -63,6 +63,7 @@
     os.environ["CFLAGS"]="-w -g"
     pass
 
+libs_pth = os.path.join(sys.prefix,"lib")
 setup (name = "cdms2",
        version=Version,
        description = "Climate Data Management System",
@@ -76,11 +77,13 @@
                                 library_dirs = cdat_info.cdunif_library_directories,
                                 libraries = cdat_info.cdunif_libraries,
                                 define_macros = macros,
-                                extra_compile_args = [ ""],
+                                runtime_library_dirs = [libs_pth],
+                                extra_compile_args = [ "-L%s"% libs_pth],
                                 ),
                       Extension('cdms2._bindex',
                                 ['Src/_bindexmodule.c', 'Src/bindex.c'],
-                                extra_compile_args = [ ""],
+                                extra_compile_args = [ "-L%s"% libs_pth],
+                                runtime_library_dirs = [libs_pth],
                                 ) 
                      ]
       )
@@ -99,6 +102,12 @@
        packages = ['regrid2'],
        package_dir = {'regrid2': 'regrid2/Lib'},
        include_dirs = [numpy.lib.utils.get_include()],
-       ext_modules = [Extension('regrid2._regrid', ['regrid2/Src/_regridmodule.c']),
-                      Extension('regrid2._scrip', ['regrid2/Src/scrip.pyf','regrid2/Src/regrid.c'])]
+       ext_modules = [Extension('regrid2._regrid', ['regrid2/Src/_regridmodule.c'],
+                                runtime_library_dirs = [libs_pth],
+                                extra_compile_args = [ "-L%s"% libs_pth],
+                                ),
+                      Extension('regrid2._scrip', ['regrid2/Src/scrip.pyf','regrid2/Src/regrid.c'],
+                                runtime_library_dirs = [libs_pth],
+                                extra_compile_args = [ "-L%s"% libs_pth],
+                          )]
       )