[CI] MDAnalysis self-dependency build fix (#4502)

IAlibay · web-flow · commit d11a1ef77628 · 2024-09-07T17:20:52.000+01:00
* Fix MDAnalysis cron CI
* Fix deployment workflow python and utility versions
* Make sure we install mdakit dependencies using no-deps
* Fix nightly wheel tests.
diff --git a/.github/actions/build-src/action.yaml b/.github/actions/build-src/action.yaml
@@ -66,6 +66,15 @@ runs:
         micromamba info
         micromamba list
 
+    - name: mda_deps
+      shell: bash -l {0}
+      run: |
+        # Install mdakit deps that depend on MDA
+        python -m pip install --no-deps \
+            waterdynamics   \
+            pathsimanalysis \
+            mdahole2
+
     - name: build_mda_main
       shell: bash -l {0}
       run: |
@@ -84,6 +93,12 @@ runs:
         fi
         python -m pip install ${BUILD_FLAGS} -v -e ./testsuite
 
+    - name: post_build_env_check
+      shell: bash -l {0}
+      run: |
+        pip list
+        micromamba list
+
     - name: build_docs
       if: ${{ inputs.build-docs == 'true' }}
       shell: bash -l {0}
diff --git a/.github/actions/setup-deps/action.yaml b/.github/actions/setup-deps/action.yaml
@@ -31,8 +31,6 @@ inputs:
     default: 'hypothesis'
   matplotlib:
     default: 'matplotlib-base'
-  mdahole2:
-    default: 'mdahole2-base'
   mda_xdrlib:
     default: 'mda-xdrlib'
   mmtf-python:
@@ -41,8 +39,6 @@ inputs:
     default: 'numpy'
   packaging:
     default: 'packaging'
-  pathsimanalysis:
-    default: 'pathsimanalysis'
   pip:
     default: 'pip'
   pytest:
@@ -53,8 +49,6 @@ inputs:
     default: 'threadpoolctl'
   tqdm:
     default: 'tqdm>=4.43.0'
-  waterdynamics:
-    default: 'waterdynamics'
   # conda-installed optional dependencies
   biopython:
     default: 'biopython>=1.80'
@@ -120,18 +114,15 @@ runs:
           ${{ inputs.griddataformats }}
           ${{ inputs.hypothesis }}
           ${{ inputs.matplotlib }}
-          ${{ inputs.mdahole2 }}
           ${{ inputs.mda_xdrlib }}
           ${{ inputs.mmtf-python }}
           ${{ inputs.numpy }}
           ${{ inputs.packaging }}
-          ${{ inputs.pathsimanalysis }}
           ${{ inputs.pip }}
           ${{ inputs.pytest }}
           ${{ inputs.scipy }}
           ${{ inputs.threadpoolctl }}
           ${{ inputs.tqdm }}
-          ${{ inputs.waterdynamics }}
         CONDA_OPT_DEPS: |
           ${{ inputs.biopython }}
           ${{ inputs.chemfiles-python }}
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -38,10 +38,10 @@ jobs:
       matrix:
         buildplat:
           - [ubuntu-22.04, manylinux_x86_64, x86_64]
-          - [macos-11, macosx_*, x86_64]
+          - [macos-12, macosx_*, x86_64]
           - [windows-2019, win_amd64, AMD64]
           - [macos-14, macosx_*, arm64]
-        python: ["cp39", "cp310", "cp311", "cp312"]
+        python: ["cp310", "cp311", "cp312"]
     defaults:
       run:
         working-directory: ./package
@@ -51,7 +51,7 @@ jobs:
           fetch-depth: 0
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.20.0
         with:
           package-dir: package
         env:
@@ -142,7 +142,7 @@ jobs:
           mv dist/MDAnalysisTests-* testsuite/dist
 
       - name: upload_source_and_wheels
-        uses: pypa/gh-action-pypi-publish@v1.10.0
+        uses: pypa/gh-action-pypi-publish@v1.10.1
         with:
           skip_existing: true
           repository_url: https://test.pypi.org/legacy/
@@ -171,7 +171,7 @@ jobs:
           mv dist/MDAnalysisTests-* testsuite/dist
 
       - name: upload_tests
-        uses: pypa/gh-action-pypi-publish@v1.10.0
+        uses: pypa/gh-action-pypi-publish@v1.10.1
         with:
           packages_dir: testsuite/dist
           skip_existing: true
diff --git a/.github/workflows/gh-ci-cron.yaml b/.github/workflows/gh-ci-cron.yaml
@@ -4,6 +4,11 @@ on:
     # 3 am Tuesdays and Fridays
     - cron: "0 3 * * 2,5"
   workflow_dispatch:
+  # Uncomment when you need to test on a PR
+  pull_request:
+    branches:
+      - develop
+
 
 concurrency:
   # Probably overly cautious group naming.
@@ -21,6 +26,7 @@ env:
   MPLBACKEND: agg
 
 jobs:
+  # a pip only, minimal deps install w/ scipy & numpy nightly upstream wheels
   numpy_and_scipy_dev:
     if: "github.repository == 'MDAnalysis/mdanalysis'"
     runs-on: ubuntu-latest
@@ -34,45 +40,51 @@ jobs:
       with:
         os-type: "ubuntu"
 
-    - name: setup_micromamba
-      uses: mamba-org/setup-micromamba@v1
-      with:
-        environment-name: mda
-        create-args: >-
-          python=3.11
-          pip
-        # using jaime's shim to avoid pulling down the cudatoolkit
-        condarc: |
-          channels:
-            - jaimergp/label/unsupported-cudatoolkit-shim
-            - conda-forge
-            - bioconda
-
-    - name: install_deps
-      uses: ./.github/actions/setup-deps
+    - uses: actions/setup-python@v4
       with:
-        micromamba: true
-        full-deps: true
+        python-version: ${{ matrix.python-version }}
 
-      # overwrite installs by picking up nightly wheels
+    # minimally install nightly wheels & core deps
     - name: nightly_wheels
       run: |
-        pip install --pre -U -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple scipy numpy networkx matplotlib pandas
+        # Nightlies: add in networkx and matplotlib because we can
+        python -m pip install --pre -U --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+            scipy           \
+            numpy           \
+            networkx        \
+            matplotlib      \
+        # Base deps
+        python -m pip install        \
+            "cython>=0.28"           \
+            packaging                \
+            "setuptools>69.4"        \
+            wheel                    \
+            "griddataformats>=0.4.0" \
+            "mmtf-python>=1.0"      \
+            "joblib>=0.12"           \
+            "tqdm>=4.43.0"           \
+            threadpoolctl            \
+            fasteners                \
+            mda-xdrlib               \
+            pytest                   \
+            pytest-xdist             \
+            pytest-timeout
+        # deps that depend on MDA
+        python -m pip install --no-deps \
+            waterdynamics   \
+            pathsimanalysis \
+            mdahole2
+
+    - name: pre_install_list_deps
+      run: python -m pip list
 
-    - name: list_deps
+    - name: build_srcs
       run: |
-        micromamba list
-        pip list
+        python -m pip install --no-build-isolation -v -e ./package
+        python -m pip install --no-build-isolation -v -e ./testsuite
 
-      # Intentionally going with setup.py builds so we can build with latest
-    - name: build_srcs
-      uses: ./.github/actions/build-src
-      with:
-        build-tests: true
-        build-docs: false
-        # We don't use build isolation because we want to ensure that we
-        # test building with brand new versions of NumPy here.
-        isolation: false
+    - name: post_install_list_deps
+      run: python -m pip list
 
     - name: run_tests
       run: |
@@ -136,7 +148,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-11]
+        os: [ubuntu-20.04, macos-12]
 
     steps:
     - uses: actions/checkout@v4
@@ -151,7 +163,7 @@ jobs:
       with:
         environment-name: mda
         create-args: >-
-          python=3.9
+          python=3.10
           pip
         condarc: |
           channels:
@@ -210,6 +222,9 @@ jobs:
       run: |
         pip install pytest-xdist pytest-timeout
 
+    - name: check env
+      run: pip list
+
     - name: run_tests
       run: |
         pytest --timeout=200 -n auto testsuite/MDAnalysisTests --disable-pytest-warnings --durations=50
@@ -218,12 +233,14 @@ jobs:
   conda-latest-release:
     # A set of runner to check that the latest conda release works as expected
     if: "github.repository == 'MDAnalysis/mdanalysis'"
-    runs-on: ${{ matrix.os }}-latest
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     strategy:
         fail-fast: false
         matrix:
-          os: [ubuntu, macos]
+          # Stick to macos-13 because some of our
+          # optional depss don't support arm64 (i.e. macos-14)
+          os: [ubuntu-latest, macos-13]
           python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
@@ -247,16 +264,16 @@ jobs:
             - conda-forge
             - bioconda
 
+    - name: install_mdanalysis
+      run: |
+          micromamba install mdanalysis mdanalysistests
+
     - name: install_deps
       uses: ./.github/actions/setup-deps
       with:
         micromamba: true
         full-deps: true
 
-    - name: install_mdanalysis
-      run: |
-          micromamba install mdanalysis mdanalysistests
-
     - name: run_tests
       run: |
           pytest --timeout=200 -n auto --pyargs MDAnalysisTests
diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml
@@ -276,6 +276,9 @@ jobs:
         python -m pip install mdanalysis-*.tar.gz
         python -m pip install mdanalysistests-*.tar.gz
 
+    - name: check install
+      run: pip list
+
     - name: run tests
       working-directory: ./dist
       run: python -m pytest --timeout=200 -n auto --pyargs MDAnalysisTests
