Skip to content

Commit a9e29ea

Browse files
dnadeau4dnadeau4
authored
Netcdf46 (#249)
* Fix python3 slice issue(setitem) and flake8 * update to libnetcdf 4.6 * try circleci unstable label * try version 2 circleci * try version 2 circleci * change workflow name * change cdtime to cdms * add certificate to circleci * add fix conda-upload in circleci 2.0 * update prep_for_build version * fix curl command * fix cicleci for cdms * use unstable channel change uvcdat for cdat * build cdms on circleci 2.0 * fix circleci config.yml * change Users/distiler to /Users/denisnadeau * add gcc_linux * add LDSHARED for linux * disable cert and py results * add gcc_linux-64 * change cdscan link * fix myproxy * add esmf and esmpy to py3 env
1 parent 1943d9e commit a9e29ea

File tree

9 files changed

+171
-232
lines changed

9 files changed

+171
-232
lines changed

‎.circleci/config.yml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
version: 2
2+
3+
checkout:
4+
post:
5+
- ./ci-support/checkout_merge_commit.sh
6+
7+
aliases:
8+
9+
- &setup_miniconda
10+
name: setup_miniconda
11+
command: |
12+
mkdir -p workspace
13+
git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat
14+
ls workspace/cdat
15+
# following will install miniconda3 under $WORKDIR/miniconda/bin
16+
python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
17+
18+
- &create_conda_env
19+
name: create_conda_env
20+
command: |
21+
export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
22+
conda config --set always_yes yes --set changeps1 no
23+
conda update -y -q conda
24+
conda config --set anaconda_upload no
25+
conda create -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient "python>3"
26+
conda create -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 "python<3"
27+
if [ $(uname) == "Linux" ]; then
28+
conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
29+
conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc_linux-64
30+
else
31+
conda install -n py3 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
32+
conda install -n py2 -c cdat/label/unstable -c cdat/label/nightly -c conda-forge gcc
33+
fi
34+
35+
- &setup_cdms
36+
name: setup_cdms
37+
command: |
38+
export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
39+
export UVCDAT_ANONYMOUS_LOG=False
40+
source activate py3
41+
mkdir $HOME/.esg
42+
echo "Get ESGF certificates"
43+
echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o $HOME/.esg/esgf.cert
44+
cp tests/dodsrccircleci $HOME/.dodsrc
45+
echo "Create .dods_cookies"
46+
curl -L -v -c $HOME/.esg/.dods_cookies --cert $HOME/.esg/esgf.cert --key $HOME/.esg/esgf.cert "https://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CM/decadal2005/mon/atmos/Amon/r1i1p1/cct/1/cct_Amon_CMCC-CM_decadal2005_r1i1p1_202601-203512.nc.dds"
47+
if [ $(uname) == "Linux" ];then
48+
export LDSHARED="$CC -shared -pthread"
49+
LDSHARED="$CC -shared -pthread" python setup.py install
50+
else
51+
python setup.py install
52+
fi
53+
source activate py2
54+
rm -rf build
55+
if [ $(uname) == "Linux" ];then
56+
export LDSHARED="$CC -shared -pthread"
57+
LDSHARED="$CC -shared -pthread" python setup.py install
58+
else
59+
python setup.py install
60+
fi
61+
62+
- &run_cdms_tests
63+
name: run_cdms_tests
64+
command: |
65+
export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
66+
export UVCDAT_ANONYMOUS_LOG=False
67+
set -e
68+
source activate py2
69+
python run_tests.py -v2
70+
PY2_RESULT=$?
71+
echo "*** py2 test result: "${PY2_RESULT}
72+
source activate py3
73+
python run_tests.py -v2
74+
PY3_RESULT=$?
75+
echo "*** py3 test result: "${PY3_RESULT}
76+
echo $PY2_RESULT > $HOME/project/$WORKDIR/py2_result.txt
77+
echo $PY3_RESULT > $HOME/project/$WORKDIR/py3_result.txt
78+
79+
- &upload_cdms
80+
name: upload_cdms
81+
command: |
82+
export PATH=$HOME/project/$WORKDIR/miniconda/bin:$PATH
83+
export LABEL="nightly";
84+
# Retrieve results
85+
PY2_RESULT=$(cat $HOME/project/$WORKDIR/py2_result.txt)
86+
PY3_RESULT=$(cat $HOME/project/$WORKDIR/py3_result.txt)
87+
echo "*** CIRCLE BRANCH: "${CIRCLE_BRANCH}
88+
echo "*** PY2_RESULT: "${PY2_RESULT}
89+
echo "*** PY3_RESULT: "${PY3_RESULT}
90+
if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then bash ./ci-support/conda_upload.sh ; fi
91+
if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH == "master" ]; then conda install -n root conda-build anaconda-client ; fi
92+
#if [ $PY2_RESULT -eq 0 -a $PY3_RESULT -eq 0 -a $CIRCLE_BRANCH != "master" ]; then
93+
if [ $CIRCLE_BRANCH != "master" ]; then
94+
export LABEL="unstable";
95+
echo "NOTE: upload LABEL="${LABEL}
96+
conda install -n root conda-build anaconda-client;
97+
bash ./ci-support/conda_upload.sh;
98+
fi
99+
100+
jobs:
101+
macos_cdms:
102+
macos:
103+
xcode: "9.2.0"
104+
environment:
105+
WORKDIR: "workspace/test_macos_cdms"
106+
steps:
107+
- checkout
108+
- run: *setup_miniconda
109+
- run: *create_conda_env
110+
- run: *setup_cdms
111+
- run: *run_cdms_tests
112+
- run: *upload_cdms
113+
- store_artifacts:
114+
path: tests_html
115+
destination: tests_html
116+
- store_artifacts:
117+
path: tests_png
118+
destination: tests_png
119+
120+
linux_cdms:
121+
machine:
122+
image: circleci/classic:latest
123+
environment:
124+
WORKDIR: "workspace/test_linux_cdms"
125+
steps:
126+
- checkout
127+
- run: *setup_miniconda
128+
- run: *create_conda_env
129+
- run: *setup_cdms
130+
- run: *run_cdms_tests
131+
- run: *upload_cdms
132+
- store_artifacts:
133+
path: tests_html
134+
destination: tests_html
135+
- store_artifacts:
136+
path: tests_png
137+
destination: tests_png
138+
139+
140+
workflows:
141+
version: 2
142+
cdms_test:
143+
jobs:
144+
- macos_cdms
145+
- linux_cdms
146+
147+

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ before_install:
2020
script:
2121
#- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
2222
- conda create -n py3 python=3.6
23-
- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient
23+
- conda install -n py3 -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl=17.2.0 nose requests flake8 myproxyclient
2424
- source activate py3
2525
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy ; fi
2626
- export UVCDAT_ANONYMOUS_LOG=False
@@ -37,7 +37,7 @@ script:
3737
- conda create -n py2 python=2.7
3838
- source activate py2
3939
- which python
40-
- conda install -c nesii/channel/dev-esmf -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
40+
- conda install -c nesii/channel/dev-esmf -c conda-forge -c cdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8
4141
- export UVCDAT_ANONYMOUS_LOG=False
4242
- python setup.py install
4343
- python run_tests.py -v2 -s

‎ci-support/circleci_mac.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎ci-support/circleci_mac_dep.sh‎

Lines changed: 0 additions & 102 deletions
This file was deleted.

‎ci-support/circleci_mac_machine_pre.sh‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎ci-support/conda_upload.sh‎

Lines changed: 17 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,34 @@
11
#!/usr/bin/env bash
22
PKG_NAME=cdms2
33
USER=cdat
4-
echo "Trying to upload conda"
5-
mkdir ${HOME}/conda-bld
6-
export CONDA_BLD_PATH=${HOME}/conda-bld
7-
export VERSION="2.12"
4+
export VERSION="3.0"
5+
echo "Trying to upload to conda"
6+
echo ""
7+
echo "Activating base env"
8+
source activate base
9+
echo "Making sure conda-build is installed"
10+
conda install "conda-build<3.10"
11+
echo "Updating conda"
12+
conda update -y -q conda
813
if [ `uname` == "Linux" ]; then
914
OS=linux-64
1015
echo "Linux OS"
11-
yum install -y wget git gcc
12-
# wget --no-check https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3.sh 2> /dev/null
13-
wget --no-check https://repo.continuum.io/miniconda/Miniconda2-4.3.30-Linux-x86_64.sh -O miniconda2.sh 2> /dev/null
14-
bash miniconda2.sh -b -p ${HOME}/miniconda
15-
export SYSPATH=$PATH
16-
export PATH=${HOME}/miniconda/bin:${SYSPATH}
17-
echo $PATH
18-
conda config --set always_yes yes --set changeps1 no
19-
conda config --set anaconda_upload false --set ssl_verify false
20-
conda install -n root -q anaconda-client "conda-build<3.3"
21-
conda install -n root gcc future
22-
which python
23-
export UVCDAT_ANONYMOUS_LOG=False
24-
BRANCH=${TRAVIS_BRANCH}
25-
# echo "Creating python 3 env"
26-
# conda create -n py3 python=3.6
27-
# conda install -n py3 -c conda-forge -c uvcdat setuptools libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient numpy
28-
# conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy
29-
# echo "Creating certificate"
30-
# source activate py3
31-
# mkdir ${HOME}/.esg
32-
# echo ${ESGF_PWD} | myproxyclient logon -s esgf-node.llnl.gov -p 7512 -t 12 -S -b -l ${ESGF_USER} -o ${HOME}/.esg/esgf.cert
33-
# ls ${HOME}/.esg
34-
# cd travis_home
35-
# ls
36-
# cp tests/dodsrc ${HOME}.dodsrc
37-
# source deactivate
3816
else
3917
echo "Mac OS"
4018
OS=osx-64
41-
BRANCH=${CIRCLE_BRANCH}
4219
fi
4320

44-
which python
45-
if [ `uname` == "Linux" ]; then
46-
conda install -n root -q anaconda-client "conda-build<3.3"
47-
else
48-
conda install -n root -q anaconda-client conda-build
49-
fi
50-
# pin conda so that conda-build does not update it
51-
#if [ `uname` == "Darwin" ]; then
52-
# echo "conda ==4.3.21" >> ~/miniconda/conda-meta/pinned # Pin conda as workaround for conda/conda#6030
53-
#fi
21+
mkdir ~/conda-bld
5422
conda config --set anaconda_upload no
23+
export CONDA_BLD_PATH=${HOME}/conda-bld
5524
echo "Cloning recipes"
56-
cd ${HOME}
57-
git clone git://github.com/UV-CDAT/conda-recipes
25+
git clone git://github.com/CDAT/conda-recipes
5826
cd conda-recipes
5927
# uvcdat creates issues for build -c uvcdat confises package and channel
6028
rm -rf uvcdat
61-
python ./prep_for_build.py -b ${BRANCH}
62-
echo "Building now"
63-
echo "use nesii/label/dev-esmf for esmf"
64-
conda build -V
65-
conda build $PKG_NAME -c nesii/label/dev-esmf -c uvcdat/label/nightly -c conda-forge -c uvcdat
66-
#
67-
# binstar config set 'false' instead of false (not quote) I have to do it manually
68-
# this is true for OSX.
69-
# binstar is changing verify_ssl to ssl_verify, but the later is not always working
70-
#
71-
# binstar config --set verify_ssl false
72-
# binstar config --set ssl_verify false
73-
#
74-
mkdir -p ~/.continuum/anaconda-client/
75-
echo "ssl_verify: false" >> ~/.continuum/anaconda-client/config.yaml
76-
echo "verify_ssl: false" >> ~/.continuum/anaconda-client/config.yaml
77-
if [ `uname` == "Darwin" ]; then
78-
# fix conda and anaconda-client conflict
79-
conda install conda==4.2.16
80-
fi
81-
anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l ${LABEL} ${CONDA_BLD_PATH}/$OS/$PKG_NAME-$VERSION.`date +%Y`*_0.tar.bz2 --force
82-
29+
export BRANCH=${CIRCLE_BRANCH}
30+
python ./prep_for_build.py -b ${BRANCH}
8331

32+
conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge --python 3.6
33+
conda build ${PKG_NAME} -c cdat/label/unstable -c conda-forge --python 2.7
34+
anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL $CONDA_BLD_PATH/$OS/${PKG_NAME}-$VERSION.`date +%Y*`0.tar.bz2 --force

0 commit comments

Comments
 (0)
⚡