TOOLS-4102 Use mise to manage our Python install

This also adds a `requirements.txt` file with the Python packages we need.

Author: autarch

CONTRIBUTING.md

@@ -12,6 +12,67 @@ If you're interested in contributing, we have a list of some suggested tickets t
 to get started on
 [here](https://jira.mongodb.org/issues/?jql=project%20%3D%20TOOLS%20AND%20labels%20%3D%20community%20and%20status%20%3D%20open)
 
+## Dev Tools (mise)
+
+We use [`mise`](https://mise.jdx.dev/) to manage dev tools (Go, Node, Python, linters, formatters,
+etc.), ensuring every developer and CI run uses the same versions.
+
+### Installing mise
+
+To install mise, run this command:
+
+```
+MISE_VERSION=$( cat ./scripts/mise-version.txt ) sh scripts/mise.run.sh
+```
+
+Then run:
+
+```
+# Enable the Go backend
+mise settings experimental=true
+# Trust the config in this repo
+mise trust
+# Install all dev tools (Go, Node, Python, linters, etc.)
+mise install
+# Install Node packages needed by eslint, prettier, etc.
+npm install
+```
+
+After `mise install`, all tools (including `go`) are available directly in your shell when you're in
+this repo's directory. See [the mise docs](https://mise.jdx.dev/) for details on shell integration.
+
+### Adding a new tool
+
+Run `mise use github:some-org/some-tool@1.2.3`. We use the `github` backend for tools that provide
+binary releases. For Go tools without binary releases, use `go:some/package@vX.Y.Z`.
+
+## Using Mise
+
+You can integrate `mise` with your shell so that dev tools are automatically added to your `PATH`
+when your working directory is in the Mongosync repo. See [the `mise` docs](https://mise.jdx.dev/)
+for more details on this.
+
+If you _don't_ want to use that integration, you can instead use
+`mise exec <list of tools> -- some-tool` to run the tools managed by `mise`, so something like
+`mise exec go -- go version`. It's important to list the tools that `mise exec` needs. Without that
+list, it will attempt to install _all_ the tools in our `mise.toml` when you run `mise exec`. This
+is generally fine locally, but can cause problems in CI, where many of the tools we use cannot be
+installed on some of the platforms we use.
+
+### In CI
+
+We do not set up the shell integration when installing `mise` in CI. That means that anything in CI
+which needs to run Go or another dev tool should use `mise exec go -- go ...` or a similar
+invocation.
+
+### Writing Code that Runs Dev Tools
+
+You may want to write code that uses these tools, either as a script for use in CI or some other
+case.
+
+All programmatic tool invocations should use `mise exec <tool>`. This will work in CI as well as
+locally.
+
 ## Getting Started
 
 1. Create a [MongoDB JIRA account](https://jira.mongodb.org/secure/Signup!default.jspa).

common.yml

@@ -491,28 +491,9 @@ functions:
           # The aws_e2e_assume_role script requires python3 with boto3.
           # Set up or use an existing python virtualenv for boto3.
           venv='venv'
-          venvNonWindows () { . "$venv"/bin/activate; python3 -m pip install boto3; }
-          venvWindows () { . "$venv"/Scripts/activate; pip install boto3; }
-
-          # default to python3 inside mongodbtoolchain
-          PATH=/opt/mongodbtoolchain/v3/bin/:$PATH
-
-          if [ -f "$venv"/bin/activate ]; then
-            echo 'activating existing virtualenv'
-            venvNonWindows
-          elif [ -f "$venv"/Scripts/activate ]; then
-            echo 'activating existing virtualenv'
-            venvWindows
-          elif virtualenv "$venv" || python -m virtualenv "$venv"; then
-            echo 'creating new virtualenv'
-            if [ -f "$venv"/bin/activate ]; then
-              echo 'activating new virtualenv'
-              venvNonWindows
-            elif [ -f "$venv"/Scripts/activate ]; then
-              echo 'activating new virtualenv'
-              venvWindows
-            fi
-          fi
+          mise exec python -- python3 -m venv "$venv"
+          . "$venv"/bin/activate
+          pip install boto3
 
           pip list
 
@@ -528,8 +509,9 @@ functions:
           set -v
           set -e
 
+          # There's no need to use mise here, since this will work with any version of Python.
           jsonkey () { python -c "import sys, json; sys.stdout.write(json.load(sys.stdin)['$1'])" < creds.json; }
-          urlencode () { python -c "import sys, urllib as ul; sys.stdout.write(ul.quote_plus('$1'))"; }
+          urlencode () { python -c "import sys, urllib.parse as ul; sys.stdout.write(ul.quote_plus('$1'))"; }
 
           USER=$(jsonkey AccessKeyId)
           USER=$(urlencode $USER)
@@ -655,8 +637,6 @@ pre:
   - command: expansions.update
     params:
       updates:
-        - key: python
-          value: "/opt/mongodbtoolchain/v4/bin/python3"
         - key: resmoke_dir
           value: "src/resmoke"
         - key: resmoke_venv_dir

mise.toml

@@ -19,6 +19,9 @@ node = "22.22.0"
 "npm:prettier" = "3.4.2"
 "npm:github-codeowners" = "0.2.1"
 
+# Python
+python = "3.12.9"
+
 [settings.node]
 # We turn off GPG verification for node because it has been flaky in CI, and also requires a local
 # GPG public key.

requirements.txt

@@ -0,0 +1,4 @@
+boto3
+pymongo==3.12.1
+pywin32; sys_platform == "win32"
+pyyaml

scripts/run_native_cert_ssl.sh

@@ -15,14 +15,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     sudo security add-trusted-cert -d -k /Library/Keychains/System.keychain jstests/libs/trusted-ca.pem
 fi
 
-PATH=/opt/mongodbtoolchain/v3/bin/:$PATH
-python="python3"
-if [ "Windows_NT" = "$OS" ]; then
-    python="py.exe -3"
-fi
-$python -m venv venv
-pip3 install pymongo==3.12.1 pyyaml
-if [ "Windows_NT" = "$OS" ]; then
-    pip3 install pywin32
-fi
-$python buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+mise exec python -- python3 -m venv venv
+. venv/bin/activate
+pip install -r ../../requirements.txt
+python3 buildscripts/resmoke.py --suite=native_cert_ssl  --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"

scripts/run_qa.sh

@@ -9,14 +9,7 @@ mv bin/* test/qa-tests/
 cd test/qa-tests
 chmod 400 jstests/libs/key*
 
-PATH=/opt/mongodbtoolchain/v3/bin/:$PATH
-python="python3"
-if [ "Windows_NT" = "$OS" ]; then
-    python="py.exe -3"
-fi
-$python -m venv venv
-pip3 install pymongo==3.12.1 pyyaml
-if [ "Windows_NT" = "$OS" ]; then
-    pip3 install pywin32
-fi
-$python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+mise exec python -- python3 -m venv venv
+. venv/bin/activate
+pip install -r ../../requirements.txt
+python3 buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"