Skip to content

Commit b4184ed

Browse files
c-dilksc-dilks
authored
ci: use container-forge images (#292)
1 parent a5754da commit b4184ed

2 files changed

Lines changed: 143 additions & 154 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 136 additions & 153 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,6 @@ defaults:
1212

1313
env:
1414
dataset: ci_test
15-
# NOTE: if any of these versions are changed, update the POM files too
16-
java_version: 21
17-
java_distribution: zulu
18-
1915

2016
concurrency:
2117
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -26,31 +22,32 @@ jobs:
2622
# build
2723
#############################################################################
2824

29-
build_timelines:
25+
build_timeline:
3026
runs-on: ubuntu-latest
27+
container:
28+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
29+
options: --user root
3130
steps:
32-
- uses: actions/checkout@v4
33-
- name: setup java
34-
uses: actions/setup-java@v4
35-
with:
36-
java-version: ${{ env.java_version }}
37-
distribution: ${{ env.java_distribution }}
38-
- name: build
39-
run: ./install.sh
40-
- name: tar # to preserve any permissions
41-
run: |
42-
tar cavf build_timeline.tar.zst target
43-
- uses: actions/upload-artifact@v4
44-
with:
45-
name: build_timelines
46-
retention-days: 1
47-
path: build*.tar.zst
31+
- uses: actions/checkout@v4
32+
- name: build
33+
run: ./install.sh
34+
- name: tar # to preserve any permissions
35+
run: |
36+
tar cavf build_timeline.tar.zst target
37+
- uses: actions/upload-artifact@v4
38+
with:
39+
name: build_timeline
40+
retention-days: 1
41+
path: build_timeline.tar.zst
4842

4943
# download test data
5044
#############################################################################
5145

5246
download_test_data:
5347
runs-on: ubuntu-latest
48+
container:
49+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
50+
options: --user root
5451
strategy:
5552
fail-fast: true
5653
matrix:
@@ -60,107 +57,92 @@ jobs:
6057
env:
6158
xrootd_file: xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/recon/${{ matrix.schema }}/validation_files.tar.zst
6259
steps:
63-
- uses: actions/cache@v4
64-
id: cache
65-
with:
66-
key: validation_files_${{ matrix.type }}
67-
path: validation_files.tar.zst
68-
lookup-only: true
69-
- name: install dependencies
70-
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
71-
run: |
72-
sudo apt -y update
73-
sudo apt -y upgrade
74-
sudo apt -y install xrootd-client
75-
- name: download
76-
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
77-
run: xrdcp ${{ env.xrootd_file }} ./
78-
- run: ls -lh .
79-
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
60+
- uses: actions/cache@v4
61+
id: cache
62+
with:
63+
key: validation_files_${{ matrix.type }}
64+
path: validation_files.tar.zst
65+
lookup-only: true
66+
- name: download
67+
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
68+
run: xrdcp ${{ env.xrootd_file }} ./
69+
- run: ls -lh .
70+
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
8071

8172
# monitoring
8273
#############################################################################
8374

8475
run_monitoring:
8576
needs:
86-
- build_timelines
77+
- build_timeline
8778
- download_test_data
8879
runs-on: ubuntu-latest
80+
container:
81+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
82+
options: --user root
8983
strategy:
9084
fail-fast: false
9185
matrix:
9286
type: [ detectors, physics ]
9387
steps:
94-
- name: setup java
95-
uses: actions/setup-java@v4
96-
with:
97-
java-version: ${{ env.java_version }}
98-
distribution: ${{ env.java_distribution }}
99-
- uses: actions/checkout@v4
100-
- uses: actions/cache/restore@v4
101-
with:
102-
key: validation_files_${{ matrix.type }}
103-
path: validation_files.tar.zst
104-
- uses: actions/download-artifact@v4
105-
with:
106-
pattern: build_*
107-
merge-multiple: true
108-
- name: untar
109-
run: ls *.tar.zst | xargs -I{} tar xavf {}
110-
- name: tree
111-
run: tree
112-
- name: run monitoring
113-
run: bin/run-monitoring.sh -d ${{env.dataset}} --findhipo --series --focus-${{matrix.type}} validation_files
114-
- name: tree slurm
115-
run: tree slurm
116-
- name: tree outfiles
117-
run: tree outfiles
118-
- uses: actions/upload-artifact@v4
119-
with:
120-
name: slurm_${{ matrix.type }}
121-
retention-days: 1
122-
path: slurm
123-
- uses: actions/upload-artifact@v4
124-
with:
125-
name: outfiles_monitoring_${{ matrix.type }}
126-
retention-days: 14
127-
path: outfiles
88+
- uses: actions/checkout@v4
89+
- uses: actions/cache/restore@v4
90+
with:
91+
key: validation_files_${{ matrix.type }}
92+
path: validation_files.tar.zst
93+
- name: download timeline build
94+
uses: actions/download-artifact@v4
95+
with:
96+
name: build_timeline
97+
- name: untar
98+
run: ls *.tar.zst | xargs -I{} tar xavf {}
99+
- run: tree
100+
- name: run monitoring
101+
run: bin/run-monitoring.sh -d ${{env.dataset}} --findhipo --series --focus-${{matrix.type}} validation_files
102+
- run: tree slurm
103+
- run: tree outfiles
104+
- uses: actions/upload-artifact@v4
105+
with:
106+
name: slurm_${{ matrix.type }}
107+
retention-days: 1
108+
path: slurm
109+
- uses: actions/upload-artifact@v4
110+
with:
111+
name: outfiles_monitoring_${{ matrix.type }}
112+
retention-days: 14
113+
path: outfiles
128114

129115
test_swifjob:
130116
needs:
131-
- build_timelines
117+
- build_timeline
132118
- download_test_data
133119
runs-on: ubuntu-latest
120+
container:
121+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
122+
options: --user root
134123
strategy:
135124
fail-fast: false
136125
matrix:
137126
type: [ detectors, physics ]
138127
steps:
139-
- name: setup java
140-
uses: actions/setup-java@v4
141-
with:
142-
java-version: ${{ env.java_version }}
143-
distribution: ${{ env.java_distribution }}
144-
- uses: actions/checkout@v4
145-
- uses: actions/cache/restore@v4
146-
with:
147-
key: validation_files_${{ matrix.type }}
148-
path: validation_files.tar.zst
149-
- uses: actions/download-artifact@v4
150-
with:
151-
pattern: build_*
152-
merge-multiple: true
153-
- name: untar
154-
run: ls *.tar.zst | xargs -I{} tar xavf {}
155-
- name: tree
156-
run: tree
157-
- name: test monitoring swifjob
158-
run: |
159-
single_rundir=$(find validation_files -mindepth 1 -maxdepth 1 -type d | head -n1)
160-
echo "single_rundir = $single_rundir"
161-
util/test-swifjob.sh $single_rundir runner --focus-${{ matrix.type }}
162-
- name: tree runner
163-
run: tree runner
128+
- uses: actions/checkout@v4
129+
- uses: actions/cache/restore@v4
130+
with:
131+
key: validation_files_${{ matrix.type }}
132+
path: validation_files.tar.zst
133+
- name: download timeline build
134+
uses: actions/download-artifact@v4
135+
with:
136+
name: build_timeline
137+
- name: untar
138+
run: ls *.tar.zst | xargs -I{} tar xavf {}
139+
- run: tree
140+
- name: test monitoring swifjob
141+
run: |
142+
single_rundir=$(find validation_files -mindepth 1 -maxdepth 1 -type d | head -n1)
143+
echo "single_rundir = $single_rundir"
144+
util/test-swifjob.sh $single_rundir runner --focus-${{ matrix.type }}
145+
- run: tree runner
164146

165147
# timeline production
166148
#############################################################################
@@ -169,41 +151,38 @@ jobs:
169151
needs:
170152
- run_monitoring
171153
runs-on: ubuntu-latest
154+
container:
155+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
156+
options: --user root
172157
strategy:
173158
fail-fast: false
174159
matrix:
175160
include:
176161
- { type: detectors, args: '-n 4 --skip-mya' }
177162
- { type: physics, args: '' }
178163
steps:
179-
- name: setup java
180-
uses: actions/setup-java@v4
181-
with:
182-
java-version: ${{ env.java_version }}
183-
distribution: ${{ env.java_distribution }}
184-
- uses: actions/checkout@v4
185-
- uses: actions/download-artifact@v4
186-
with:
187-
pattern: outfiles_*
188-
merge-multiple: true
189-
path: outfiles
190-
- uses: actions/download-artifact@v4
191-
with:
192-
pattern: build_*
193-
merge-multiple: true
194-
- name: untar
195-
run: ls *.tar.zst | xargs -I{} tar xavf {}
196-
- name: tree
197-
run: tree
198-
- name: run timelines
199-
run: bin/run-${{matrix.type}}-timelines.sh -d ${{env.dataset}} ${{matrix.args}}
200-
- name: tree outfiles
201-
run: tree outfiles
202-
- uses: actions/upload-artifact@v4
203-
with:
204-
name: outfiles_timelines_${{ matrix.type }}
205-
retention-days: 14
206-
path: outfiles
164+
- uses: actions/checkout@v4
165+
- name: download outfiles
166+
uses: actions/download-artifact@v4
167+
with:
168+
pattern: outfiles_*
169+
merge-multiple: true
170+
path: outfiles
171+
- name: download timeline build
172+
uses: actions/download-artifact@v4
173+
with:
174+
name: build_timeline
175+
- name: untar
176+
run: ls *.tar.zst | xargs -I{} tar xavf {}
177+
- run: tree
178+
- name: run timelines
179+
run: bin/run-${{matrix.type}}-timelines.sh -d ${{env.dataset}} ${{matrix.args}}
180+
- run: tree outfiles
181+
- uses: actions/upload-artifact@v4
182+
with:
183+
name: outfiles_timelines_${{ matrix.type }}
184+
retention-days: 14
185+
path: outfiles
207186

208187
# timeline deployment
209188
#############################################################################
@@ -213,30 +192,34 @@ jobs:
213192
needs:
214193
- run_timelines
215194
runs-on: ubuntu-latest
195+
container:
196+
image: codecr.jlab.org/hallb/clas12/container-forge/base:latest
197+
options: --user root
216198
steps:
217-
- uses: actions/checkout@v4
218-
- uses: actions/download-artifact@v4
219-
with:
220-
pattern: outfiles_*
221-
merge-multiple: true
222-
path: outfiles
223-
- uses: actions/download-artifact@v4
224-
with:
225-
pattern: build_*
226-
merge-multiple: true
227-
- name: untar
228-
run: ls *.tar.zst | xargs -I{} tar xavf {}
229-
- name: tree outfiles
230-
run: |
231-
rm -r outfiles/${{ env.dataset }}/log
232-
echo "# Output File Tree" >> $GITHUB_STEP_SUMMARY
233-
echo '```' >> $GITHUB_STEP_SUMMARY
234-
tree outfiles | xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY
235-
echo '```' >> $GITHUB_STEP_SUMMARY
236-
- name: deploy
237-
run: bin/deploy-timelines.sh -d ${{env.dataset}} -c -t web
238-
- uses: actions/upload-artifact@v4
239-
with:
240-
name: web
241-
retention-days: 14
242-
path: web
199+
- uses: actions/checkout@v4
200+
- name: download outfiles
201+
uses: actions/download-artifact@v4
202+
with:
203+
pattern: outfiles_*
204+
merge-multiple: true
205+
path: outfiles
206+
- name: download timeline build
207+
uses: actions/download-artifact@v4
208+
with:
209+
name: build_timeline
210+
- name: untar
211+
run: ls *.tar.zst | xargs -I{} tar xavf {}
212+
- name: tree outfiles
213+
run: |
214+
rm -r outfiles/${{ env.dataset }}/log
215+
echo "# Output File Tree" >> $GITHUB_STEP_SUMMARY
216+
echo '```' >> $GITHUB_STEP_SUMMARY
217+
tree outfiles | xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY
218+
echo '```' >> $GITHUB_STEP_SUMMARY
219+
- name: deploy
220+
run: bin/deploy-timelines.sh -d ${{env.dataset}} -c -t web
221+
- uses: actions/upload-artifact@v4
222+
with:
223+
name: web
224+
retention-days: 14
225+
path: web

‎libexec/environ.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,14 @@ log_config=logging # set to 'logging' for quiet, or to 'debug' for verbose
1111
[ -z "${BASH_SOURCE[0]}" ] && this_env=$0 || this_env=${BASH_SOURCE[0]}
1212
export TIMELINESRC=$(realpath $(dirname $this_env)/..)
1313

14-
# RCDB
14+
# database URIs
1515
[ -z "${RCDB_CONNECTION-}" ] && export RCDB_CONNECTION=mysql://rcdb@clasdb.jlab.org/rcdb
16+
[ -z "${CCDB_CONNECTION-}" ] && export CCDB_CONNECTION=mysql://clas12reader@clasdb-farm.jlab.org/clas12
17+
# overrides for CI
18+
if ${GITHUB_ACTIONS:-false}; then
19+
export RCDB_CONNECTION=mysql://rcdb@clasdb.jlab.org/rcdb
20+
export CCDB_CONNECTION=mysql://clas12reader@clasdb.jlab.org/clas12
21+
fi
1622

1723
# java options
1824
timeline_java_opts=(

0 commit comments

Comments
 (0)
⚡