Skip to content

Commit 28991ce

Browse files
adcroftadcroft
committed
Pipeline: major refactor
- Makes pipeline robust to changes in FMS, SIS2, etc. by doing clean builds. - Also makes system robust to multiple PRs being tested in parallel. - Used a cache directory containing tar files with artifacts - gitlab artifacts are transferred to the server and these artifacts are far too big for that. - Separates layout test from symmetric - non-symmetric comparison. - Adds gnu restart test. - Adds gnu static test. Note: I elected to use gnu rather than intel for the restart/static tests because lustre file errors seem to be most often associated with the intel executable - I've no idea why that would be.
1 parent b8ab30d commit 28991ce

1 file changed

Lines changed: 117 additions & 53 deletions

File tree

‎.gitlab-ci.yml‎

Lines changed: 117 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,138 +1,202 @@
11
stages:
2-
- merge
3-
- setup
2+
- merge+setup
43
- builds
54
- run
65
- tests
6+
- cleanup
77

8-
# Moves to re-used workspace
8+
# Merges MOM6 with dev/gfdl. Changes directory to test directory, if it exists.
99
before_script:
1010
- MOM6_SRC=$CI_PROJECT_DIR
11-
- WORKSPACE=`pwd | sed 's:/[0-9]/:/:'`
12-
- git pull --no-edit https://github.com/NOAA-GFDL/MOM6.git dev/gfdl
13-
- test -d $WORKSPACE/tests && cd $WORKSPACE/tests
11+
- CACHE_DIR=/lustre/f1/oar.gfdl.ogrp-account/runner/cache/
12+
- git pull --no-edit https://github.com/NOAA-GFDL/MOM6.git dev/gfdl && git submodule init && git submodule update
13+
- pwd ; ls
1414

15-
# Merge
15+
# Tests that merge with dev/gfdl works.
1616
merge:
17-
stage: merge
17+
stage: merge+setup
1818
tags:
1919
- ncrc3
2020
script:
21-
- cd $CI_PROJECT_DIR
21+
- pwd ; ls
2222
- git pull --no-edit https://github.com/NOAA-GFDL/MOM6.git dev/gfdl
2323

2424
# Clones regression repo, if necessary, pulls latest of everything, and sets up working space
2525
setup:
26-
stage: setup
26+
stage: merge+setup
2727
tags:
2828
- ncrc3
2929
script:
30-
- mkdir -p $WORKSPACE && cd $WORKSPACE
31-
- ln -sf $MOM6_SRC/{config_src,src,pkg} .
32-
- test -d tests || (git clone --recursive http://gitlab.gfdl.noaa.gov/ogrp/Gaea_c3-stats-MOM6-examples.git tests)
33-
- cd tests
34-
- git checkout . && git checkout dev/gfdl
35-
- git pull
36-
- git status
37-
- bash Gitlab/before_script.sh
38-
- (cd MOM6-examples/src/mkmf && git pull https://github.com/adcroft/mkmf.git add_coverage_mode)
30+
- pwd ; ls
31+
# Clone regressions directory
32+
- git clone --recursive http://gitlab.gfdl.noaa.gov/ogrp/Gaea_c3-stats-MOM6-examples.git tests && cd tests
33+
# Install / update testing scripts
34+
- git clone https://github.com/adcroft/MRS.git MRS
35+
# Update MOM6-examples and submodules
36+
- (cd MOM6-examples && git checkout . && git checkout dev/gfdl && git pull && git submodule init && git submodule update)
37+
- test -d MOM6-examples/src/LM3 || make -f MRS/Makefile.clone clone_gfdl -s
38+
- make -f MRS/Makefile.clone MOM6-examples/.datasets -s
39+
#- (cd MOM6-examples/src/mkmf && git pull https://github.com/adcroft/mkmf.git add_coverage_mode)
40+
- env > gitlab_session.log
41+
- cd ../ ; time tar zcf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz tests
3942

4043
# Compiles
4144
gnu:repro:
4245
stage: builds
4346
tags:
4447
- ncrc3
4548
script:
46-
- make -f Gitlab/Makefile.build MOM6_SRC=../ build_gnu -s -j
47-
- make -f Gitlab/Makefile.build MOM6_SRC=../ static_gnu -s -j #coverage_gnu
49+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
50+
- time make -f MRS/Makefile.build MOM6_SRC=../ build_gnu -s -j
51+
- time make -f MRS/Makefile.build MOM6_SRC=../ static_gnu -s -j
52+
- time tar zvcf $CACHE_DIR/build-gnu-repro-$CI_PIPELINE_ID.tgz `find build/gnu -name MOM6`
4853

4954
intel:repro:
5055
stage: builds
5156
tags:
5257
- ncrc3
5358
script:
54-
- make -f Gitlab/Makefile.build MOM6_SRC=../ build_intel -s -j
59+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
60+
- make -f MRS/Makefile.build MOM6_SRC=../ build_intel -s -j
61+
- time tar zvcf $CACHE_DIR/build-intel-repro-$CI_PIPELINE_ID.tgz `find build/intel -name MOM6`
5562

5663
pgi:repro:
5764
stage: builds
5865
tags:
5966
- ncrc3
6067
script:
61-
- make -f Gitlab/Makefile.build MOM6_SRC=../ build_pgi -s -j
68+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
69+
- make -f MRS/Makefile.build MOM6_SRC=../ build_pgi -s -j
70+
- time tar zvcf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz `find build/pgi -name MOM6`
6271

6372
gnu:debug:
6473
stage: builds
6574
tags:
6675
- ncrc3
6776
script:
68-
- make -f Gitlab/Makefile.build MOM6_SRC=../ debug_gnu -s -j
77+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
78+
- make -f MRS/Makefile.build MOM6_SRC=../ debug_gnu -s -j
79+
- time tar zvcf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz `find build/gnu -name MOM6`
6980

7081
# Runs
7182
run:
7283
stage: run
7384
tags:
7485
- ncrc3
7586
script:
76-
- rm -f MOM6-examples/*.tar
77-
- bash Gitlab/run_stage.sh
87+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
88+
- time tar zxf $CACHE_DIR/build-gnu-repro-$CI_PIPELINE_ID.tgz
89+
- time tar zxf $CACHE_DIR/build-intel-repro-$CI_PIPELINE_ID.tgz
90+
- time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
91+
# time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
92+
- echo "make -f MRS/Makefile.tests all -B" > job.sh
93+
- msub -l partition=c3,nodes=29,walltime=00:24:00,qos=norm -q debug -S /bin/tcsh -j oe -A gfdl_o -z -o log.$CI_PIPELINE_ID -N mom6_regression -K job.sh
94+
- cat log.$CI_PIPELINE_ID
95+
- time tar zvcf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz *.tar.gz
7896

7997
# Tests
80-
gnu:regressions:
98+
gnu:non-symmetric:
8199
stage: tests
82100
tags:
83101
- ncrc3
84102
script:
85-
- cd regressions && git checkout .
86-
- md5sum `find . -name ocean.stats.gnu` > all_gnu.md5 && find . -name ocean.stats.gnu -exec rm {} \;
87-
- echo "Regression results for non-symmetric gnu executables"
88-
- tar xf ../MOM6-examples/non_symmetric_gnu.tar && md5sum -c all_gnu.md5 && rm all_gnu.md5
103+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
104+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
105+
- make -f MRS/Makefile.tests gnu_non_symmetric
89106

90-
intel:regressions:
107+
intel:non-symmetric:
91108
stage: tests
92109
tags:
93110
- ncrc3
94111
script:
95-
- cd regressions && git checkout .
96-
- md5sum `find . -name ocean.stats.intel` > all_intel.md5 && find . -name ocean.stats.intel -exec rm {} \;
97-
- echo "Regression results for non-symmetric intel executables"
98-
- tar xf ../MOM6-examples/non_symmetric_intel.tar && md5sum -c all_intel.md5 && rm all_intel.md5
112+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
113+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
114+
- make -f MRS/Makefile.tests intel_non_symmetric
99115

100-
pgi:regressions:
116+
pgi:non-symmetric:
101117
stage: tests
102118
tags:
103119
- ncrc3
104120
script:
105-
- cd regressions && git checkout .
106-
- md5sum `find . -name ocean.stats.pgi` > all_pgi.md5 && find . -name ocean.stats.pgi -exec rm {} \;
107-
- echo "Regression results for non-symmetric pgi executables"
108-
- tar xf ../MOM6-examples/non_symmetric_pgi.tar && md5sum -c all_pgi.md5 && rm all_pgi.md5
121+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
122+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
123+
- make -f MRS/Makefile.tests pgi_non_symmetric
109124

110125
gnu:symmetric:
111126
stage: tests
112127
tags:
113128
- ncrc3
114129
script:
115-
- mkdir -p _gnu && cd _gnu
116-
- tar xf ../MOM6-examples/non_symmetric_gnu.tar && md5sum `find . -name ocean.stats.gnu` > all_gnu.md5 && find . -name ocean.stats.gnu -exec rm {} \;
117-
- echo "Regression results for symmetric executables"
118-
- tar xf ../MOM6-examples/symmetric_gnu.tar && md5sum -c all_gnu.md5 && rm all_gnu.md5
130+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
131+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
132+
- make -f MRS/Makefile.tests gnu_symmetric
119133

120134
intel:symmetric:
121135
stage: tests
122136
tags:
123137
- ncrc3
124138
script:
125-
- mkdir -p _intel && cd _intel
126-
- tar xf ../MOM6-examples/non_symmetric_intel.tar && md5sum `find . -name ocean.stats.intel` > all_intel.md5 && find . -name ocean.stats.intel -exec rm {} \;
127-
- echo "Regression results for symmetric executables"
128-
- tar xf ../MOM6-examples/symmetric_intel.tar && md5sum -c all_intel.md5 && rm all_intel.md5
139+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
140+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
141+
- make -f MRS/Makefile.tests intel_symmetric
129142

130143
pgi:symmetric:
131144
stage: tests
132145
tags:
133146
- ncrc3
134147
script:
135-
- mkdir -p _pgi && cd _pgi
136-
- tar xf ../MOM6-examples/non_symmetric_pgi.tar && md5sum `find . -name ocean.stats.pgi` > all_pgi.md5 && find . -name ocean.stats.pgi -exec rm {} \;
137-
- echo "Regression results for symmetric executables"
138-
- tar xf ../MOM6-examples/symmetric_pgi.tar && md5sum -c all_pgi.md5 && rm all_pgi.md5
148+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
149+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
150+
- make -f MRS/Makefile.tests pgi_symmetric
151+
152+
gnu:layout:
153+
stage: tests
154+
tags:
155+
- ncrc3
156+
script:
157+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
158+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
159+
- make -f MRS/Makefile.tests gnu_layout
160+
161+
intel:layout:
162+
stage: tests
163+
tags:
164+
- ncrc3
165+
script:
166+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
167+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
168+
- make -f MRS/Makefile.tests intel_layout
169+
170+
pgi:layout:
171+
stage: tests
172+
tags:
173+
- ncrc3
174+
script:
175+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
176+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
177+
- make -f MRS/Makefile.tests pgi_layout
178+
179+
gnu:static:
180+
stage: tests
181+
tags:
182+
- ncrc3
183+
script:
184+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
185+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
186+
- make -f MRS/Makefile.tests gnu_static
187+
188+
gnu:restart:
189+
stage: tests
190+
tags:
191+
- ncrc3
192+
script:
193+
- time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
194+
- time tar zxf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz
195+
- make -f MRS/Makefile.tests gnu_check_restarts
196+
197+
cleanup:
198+
stage: cleanup
199+
tags:
200+
- ncrc3
201+
script:
202+
- rm $CACHE_DIR/*_$CI_PIPELINE_ID.tgz

0 commit comments

Comments
 (0)
⚡