Performance monitoring; GitHub integration

This patch introduces two new testing targets to the verification suite
based on a small configuration based on the `benchmark` regression test.

The profile test is saved a `p0` in `.testing`.  Future tests can be
included if appropriate.

The new targets:

* `make profile`: Run the model and record the FMS timings.

* `make perf`: Run the model through the `perf` tool and record timings
  for the resolvable functions (as symbols).

In both cases, the timings are converted to JSON output files and the
top results are reported to stdout, and readable in GitHub actions
output.  It can also be run locally.

Support Python scripts have been included to do this work.  This will
require a functional Python environment.

Some system and configuration data is logged alongside the timings, but
this is still rather incomplete and needs some further planning.

Times are compared to the target build (usually dev/gfdl).  ANSI
terminal highlighting (i.e. color) is to used to highlight excessive
differences.

Current issues:

- Model configuration

- GitHub timings are still rather unreliable, and should currently only
  be treated as crude estimates.  This should be considered a work in
  progress.

- The GitHub profiling rule still builds the standard configuration,
  evem though it is unused.

- Additional tools are required to push the timings to some database,
  either a local sqlite3 or an external one.

Author: marshallward

.github/actions/testing-setup/action.yml

@@ -33,6 +33,7 @@ runs:
         echo "::group::Install linux packages"
         sudo apt-get update
         sudo apt-get install netcdf-bin libnetcdf-dev libnetcdff-dev mpich libmpich-dev
+        sudo apt-get install linux-tools-common
         echo "::endgroup::"
 
     - name: Compile FMS library

.github/workflows/perfmon.yml

@@ -0,0 +1,36 @@
+name: Performance Monitor
+
+on: [pull_request]
+
+jobs:
+  build-test-perfmon:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile optimized models
+      run: >-
+        make -j build.prof
+        MOM_TARGET_SLUG=$GITHUB_REPOSITORY
+        MOM_TARGET_LOCAL_BRANCH=$GITHUB_BASE_REF
+        DO_REGRESSION_TESTS=true
+
+    - name: Generate profile data
+      run: >-
+        pip install f90nml &&
+        make profile
+        DO_REGRESSION_TESTS=true
+
+    - name: Generate perf data
+      run: |
+        sudo sysctl -w kernel.perf_event_paranoid=2
+        make perf DO_REGRESSION_TESTS=true

.testing/Makefile

@@ -34,6 +34,7 @@
 # Build configuration:
 #   FCFLAGS_DEBUG       Testing ("debug") compiler flags
 #   FCFLAGS_REPRO       Production ("repro") compiler flags
+#   FCFLAGS_OPT			Aggressive optimization compiler flags
 #   FCFLAGS_INIT        Variable initialization flags
 #   FCFLAGS_COVERAGE    Code coverage flags
 #
@@ -72,6 +73,7 @@ export MPIFC
 # NOTE: FMS will be built using FCFLAGS_DEBUG
 FCFLAGS_DEBUG ?= -g -O0
 FCFLAGS_REPRO ?= -g -O2
+FCFLAGS_OPT	?= -g -O3 -mavx -fno-omit-frame-pointer
 FCFLAGS_INIT ?=
 FCFLAGS_COVERAGE ?=
 # Additional notes:
@@ -96,6 +98,7 @@ DO_REPRO_TESTS ?=
 # Time measurement (configurable by the CI)
 TIME ?= time
 
+
 #---
 # Dependencies
 DEPS = deps
@@ -122,6 +125,11 @@ ifeq ($(DO_REPRO_TESTS), true)
   TESTS += repros
 endif
 
+# Profiling
+ifeq ($(DO_PROFILE), false)
+  BUILDS += opt opt_target
+endif
+
 # The following variables are configured by Travis:
 #   DO_REGRESSION_TESTS: true if $(TRAVIS_PULL_REQUEST) is a PR number
 #   MOM_TARGET_SLUG: TRAVIS_REPO_SLUG
@@ -195,6 +203,7 @@ endif
 .PHONY: all build.regressions
 all: $(foreach b,$(BUILDS),build/$(b)/MOM6) $(VENV_PATH)
 build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6)
+build.prof: $(foreach b,opt opt_target,build/$(b)/MOM6)
 
 # Executable
 BUILD_TARGETS = MOM6 Makefile path_names
@@ -217,6 +226,7 @@ PATH_FMS = PATH="${PATH}:../../$(DEPS)/bin"
 SYMMETRIC_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(COVERAGE) $(FCFLAGS_FMS)"
 ASYMMETRIC_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
 REPRO_FCFLAGS := FCFLAGS="$(FCFLAGS_REPRO) $(FCFLAGS_FMS)"
+OPT_FCFLAGS := FCFLAGS="$(FCFLAGS_OPT) $(FCFLAGS_FMS)"
 OPENMP_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
 TARGET_FCFLAGS := FCFLAGS="$(FCFLAGS_DEBUG) $(FCFLAGS_INIT) $(FCFLAGS_FMS)"
 
@@ -230,6 +240,8 @@ build/asymmetric/Makefile: MOM_ENV=$(PATH_FMS) $(ASYMMETRIC_FCFLAGS) $(MOM_LDFLA
 build/repro/Makefile: MOM_ENV=$(PATH_FMS) $(REPRO_FCFLAGS) $(MOM_LDFLAGS)
 build/openmp/Makefile: MOM_ENV=$(PATH_FMS) $(OPENMP_FCFLAGS) $(MOM_LDFLAGS)
 build/target/Makefile: MOM_ENV=$(PATH_FMS) $(TARGET_FCFLAGS) $(MOM_LDFLAGS)
+build/opt/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
+build/opt_target/Makefile: MOM_ENV=$(PATH_FMS) $(OPT_FCFLAGS) $(MOM_LDFLAGS)
 build/coupled/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
 build/nuopc/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
 build/mct/Makefile: MOM_ENV=$(PATH_FMS) $(SYMMETRIC_FCFLAGS) $(SYMMETRIC_LDFLAGS)
@@ -240,12 +252,15 @@ build/asymmetric/Makefile: MOM_ACFLAGS=--enable-asymmetric
 build/repro/Makefile: MOM_ACFLAGS=
 build/openmp/Makefile: MOM_ACFLAGS=--enable-openmp
 build/target/Makefile: MOM_ACFLAGS=
+build/opt/Makefile: MOM_ACFLAGS=
+build/opt_target/Makefile: MOM_ACFLAGS=
 build/coupled/Makefile: MOM_ACFLAGS=--with-driver=coupled_driver
 build/nuopc/Makefile: MOM_ACFLAGS=--with-driver=nuopc_driver
 build/mct/Makefile: MOM_ACFLAGS=--with-driver=mct_driver
 
 # Fetch regression target source code
 build/target/Makefile: | $(TARGET_CODEBASE)
+build/opt_target/Makefile: | $(TARGET_CODEBASE)
 
 
 # Define source code dependencies
@@ -276,7 +291,8 @@ build/%/Makefile: ../ac/configure ../ac/Makefile.in $(DEPS)/lib/libFMS.a $(MKMF)
 
 
 # Fetch the regression target codebase
-build/target/Makefile: $(TARGET_CODEBASE)/ac/configure $(DEPS)/lib/libFMS.a $(MKMF) $(LIST_PATHS)
+build/target/Makefile build/opt_target/Makefile: \
+		$(TARGET_CODEBASE)/ac/configure $(DEPS)/lib/libFMS.a $(MKMF) $(LIST_PATHS)
 	mkdir -p $(@D)
 	cd $(@D) \
 	  && $(MOM_ENV) ../../$(TARGET_CODEBASE)/ac/configure $(MOM_ACFLAGS) \
@@ -620,6 +636,64 @@ test.summary:
 	fi
 
 
+#---
+# Profiling
+# XXX: This is experimental work to track, log, and report changes in runtime
+PCONFIGS = p0
+
+.PHONY: profile
+profile: $(foreach p,$(PCONFIGS), prof.$(p))
+
+.PHONY: prof.p0
+prof.p0: work/p0/opt/clocks.json work/p0/opt_target/clocks.json
+	python tools/compare_clocks.py $^
+
+work/p0/%/clocks.json: work/p0/%/std.out
+	python tools/parse_fms_clocks.py -d $(@D) $^ > $@
+
+work/p0/opt/std.out: build/opt/MOM6
+work/p0/opt_target/std.out: build/opt_target/MOM6
+
+work/p0/%/std.out:
+	mkdir -p $(@D)
+	cp -RL p0/* $(@D)
+	mkdir -p $(@D)/RESTART
+	echo -e "" > $(@D)/MOM_override
+	cd $(@D) \
+	  && $(MPIRUN) -n 1 ../../../$< 2> std.err > std.out
+
+#---
+# Same but with perf
+
+# TODO: This expects the -e flag, can I handle it in the command?
+PERF_EVENTS ?=
+
+.PHONY: perf
+perf: $(foreach p,$(PCONFIGS), perf.$(p))
+
+.PHONY: prof.p0
+perf.p0: work/p0/opt/profile.json work/p0/opt_target/profile.json
+	python tools/compare_perf.py $^
+
+work/p0/%/profile.json: work/p0/%/perf.data
+	python tools/parse_perf.py -f $< > $@
+
+work/p0/opt/perf.data: build/opt/MOM6
+work/p0/opt_target/perf.data: build/opt_target/MOM6
+
+work/p0/%/perf.data:
+	mkdir -p $(@D)
+	cp -RL p0/* $(@D)
+	mkdir -p $(@D)/RESTART
+	echo -e "" > $(@D)/MOM_override
+	cd $(@D) \
+	  && perf record \
+	    -F 3999 \
+	    ${PERF_EVENTS} \
+	    ../../../$< 2> std.perf.err > std.perf.out \
+	  || cat std.perf.err
+
+
 #----
 # NOTE: These tests assert that we are in the .testing directory.