Make mimalloc opt-in by default

This commit switches the mimalloc feature to be opt-in by default
everywhere. Since building mimalloc requires a C compiler and this does
significantly change the requirements for building Qiskit from source,
making it opt-in for people that want it is the best course of action
for now. This commit adds a new environment variable
`QISKIT_BUILD_WITH_MIMALLOC=1` which is used to enable building mimalloc
when building Qiskit. The CI jobs are updated to mix mimalloc and not in
Python test jobs and also to always enable mimalloc in wheel builds for
release.

Author: mtreinish

.github/workflows/branch-protection.yml

@@ -106,6 +106,7 @@ jobs:
             install-optionals: true
             install-from-sdist: false
             qiskit-pycache: false
+            mimalloc: false
           - id: "Python newest"
             python: ${{ needs.config.outputs.python-new }}
             install-optionals: false
@@ -127,6 +128,7 @@ jobs:
           - id: "Python oldest"
             python: ${{ needs.config.outputs.python-old }}
             install-optionals: true
+            mimalloc: false
           - id: "Python newest"
             python: ${{ needs.config.outputs.python-new }}
             install-optionals: false
@@ -146,6 +148,7 @@ jobs:
           - id: "Python oldest"
             python: ${{ needs.config.outputs.python-old }}
             install-optionals: true
+            mimalloc: false
           - id: "Python newest"
             python: ${{ needs.config.outputs.python-new }}
             install-optionals: false

.github/workflows/test-linux.yml

@@ -29,6 +29,11 @@ on:
           structures when building Qiskit.
         type: boolean
         default: true
+      mimalloc:
+        description: >
+          Whether to build with mimalloc or not
+        type: boolean
+        default: true
 
 jobs:
   unit-tests:
@@ -61,6 +66,9 @@ jobs:
       - name: Set gate-caching settings for build
         if: ${{ !inputs.qiskit-pycache }}
         run: 'echo "QISKIT_NO_CACHE_GATES=1" >> $GITHUB_ENV'
+      - name: Set mimalloc for build
+        if: ${{ inputs.mimalloc }}
+        run: 'echo "QISKIT_BUILD_WITH_MIMALLOC=1" >> $GITHUB_ENV'
       - name: Install Qiskit from sdist
         if: ${{ inputs.install-from-sdist }}
         run: |

.github/workflows/test-mac.yml

@@ -8,18 +8,22 @@ on:
           Python version to currently test
         type: string
         required: true
-
       install-optionals:
         description: >
           Decides whether we install optyional dependencies
         type: boolean
         default: false
-
       runner:
         description: >
           Describes the system this workflow should run on.
         type: string
         required: true
+      mimalloc:
+        description: >
+          Whether to build with mimalloc or not
+        type: boolean
+        default: true
+
 jobs:
   tests-mac:
     name: ${{ inputs.runner }} - Python ${{ inputs.python-version }}
@@ -42,6 +46,9 @@ jobs:
             stestr | "${{ runner.os }}"
             stestr
           path: .stestr
+      - name: Set mimalloc for build
+        if: ${{ inputs.mimalloc }}
+        run: 'echo "QISKIT_BUILD_WITH_MIMALLOC=1" >> $GITHUB_ENV'
       - name: "Install dependencies"
         run: |
           set -e

.github/workflows/test-windows.yml

@@ -8,18 +8,21 @@ on:
           Python version to currently test
         type: string
         required: true
-
       runner:
         description: >
           Describes the system this workflow should run on.
         type: string
         required: true
-
       install-optionals:
         description: >
           Decides whether we install optyional dependencies
         type: boolean
         default: false
+      mimalloc:
+        description: >
+          Whether to build with mimalloc or not
+        type: boolean
+        default: true
 jobs:
   test-windows:
     name: ${{ inputs.runner }} - Python ${{ inputs.python-version }}
@@ -42,6 +45,9 @@ jobs:
             stestr | "${{ runner.os }}"
             stestr
           path: .stestr
+      - name: Set mimalloc for build
+        if: ${{ inputs.mimalloc }}
+        run: 'echo "QISKIT_BUILD_WITH_MIMALLOC=1" >> $GITHUB_ENV'
       - name: Install dependencies
         run: |
           python -m venv test-job

.github/workflows/wheels-build.yml

@@ -108,9 +108,9 @@ jobs:
           cat >>"$GITHUB_ENV" <<EOF
           CIBW_BEFORE_ALL_LINUX=dnf install -y wget && {package}/tools/install_rust.sh
           CIBW_BEFORE_BUILD=bash ./tools/build_pgo.sh $PGO_WORK_DIR $PGO_OUT_PATH
-          CIBW_ENVIRONMENT=RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function'
-          CIBW_ENVIRONMENT_MACOS=MACOSX_DEPLOYMENT_TARGET='10.12' RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function'
-          CIBW_ENVIRONMENT_LINUX=RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function' PATH="\$PATH:\$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI="true"
+          CIBW_ENVIRONMENT=RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function' QISKIT_BUILD_WITH_MIMALLOC=1
+          CIBW_ENVIRONMENT_MACOS=MACOSX_DEPLOYMENT_TARGET='10.12' RUSTUP_TOOLCHAIN=stable RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function' QISKIT_BUILD_WITH_MIMALLOC=1
+          CIBW_ENVIRONMENT_LINUX=RUSTUP_TOOLCHAIN=stable QISKIT_BUILD_WITH_MIMALLOC=1 RUSTFLAGS='-Cprofile-use=$PGO_OUT_PATH -Cllvm-args=-pgo-warn-missing-function' PATH="\$PATH:\$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI="true"
           EOF
         env:
           PGO_WORK_DIR: ${{ github.workspace }}/pgo-data

CONTRIBUTING.md

@@ -166,7 +166,11 @@ to `build_rust` overrides this default.
 ### Compile time options
 
 When building qiskit from source there are options available to control how
-Qiskit is built. Right now the only option is if you set the environment
+Qiskit is built. The first option is if you set the
+environment variable `QISKIT_BUILD_WITH_MIMALLOC=1` this will enable using
+[mimalloc](https://github.com/microsoft/mimalloc) as the global allocator for
+Qiskit. This improves the runtime performance of Qiskit but will require having
+a C compiler installed when building Qiskit. If you set the environment
 variable `QISKIT_NO_CACHE_GATES=1` this will disable runtime caching of
 Python gate objects when accessing them from a `QuantumCircuit` or `DAGCircuit`.
 This makes a tradeoff between runtime performance for Python access and memory

Makefile

@@ -14,6 +14,10 @@ ifneq ($(OS), Windows_NT)
 	OS := $(shell uname -s)
 endif
 
+ifeq ($(QISKIT_BUILD_WITH_MIMALLOC), 1)
+	MIMALLOC := --features=mimalloc
+endif
+
 .PHONY: default ruff env lint lint-incr style black test test_randomized pytest pytest_randomized test_ci coverage coverage_erase clean
 
 default: style lint-incr test ;
@@ -135,7 +139,7 @@ fix_cformat:
 # instead.
 .PHONY: build-clib build-clib-release build-clib-dev
 build-clib:
-	cargo rustc -p qiskit-cext --features=mimalloc --crate-type cdylib ${C_LIB_CARGO_FLAGS} -- ${C_LIB_RUSTC_FLAGS}
+	cargo rustc -p qiskit-cext ${MIMALLOC} --crate-type cdylib ${C_LIB_CARGO_FLAGS} -- ${C_LIB_RUSTC_FLAGS}
 build-clib-release: C_LIB_CARGO_FLAGS=--release
 build-clib-release: build-clib
 build-clib-dev: C_LIB_CARGO_FLAGS=--profile dev

pyproject.toml

@@ -299,16 +299,16 @@ before-test = "pip install --only-binary=numpy,scipy numpy scipy -c constraints.
 # specific job override is needed. For example tier 1 platforms locally override
 # the before-build and environment configuration to enable PGO,
 # see: .github/workflows/wheels.yml for the jobs where this is done
-environment = 'RUSTUP_TOOLCHAIN="stable"'
+environment = 'RUSTUP_TOOLCHAIN="stable" QISKIT_BUILD_WITH_MIMALLOC=1'
 
 [tool.cibuildwheel.linux]
 before-all = "yum install -y wget && {package}/tools/install_rust.sh"
-environment = 'PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI="true" RUSTUP_TOOLCHAIN="stable"'
+environment = 'PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI="true" RUSTUP_TOOLCHAIN="stable" QISKIT_BUILD_WITH_MIMALLOC=1'
 repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && pipx run abi3audit --strict --report {wheel}"
 
 [tool.cibuildwheel.macos]
 repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} && pipx run abi3audit --strict --report {wheel}"
-environment = "MACOSX_DEPLOYMENT_TARGET='10.12' RUSTUP_TOOLCHAIN=stable"
+environment = "MACOSX_DEPLOYMENT_TARGET='10.12' RUSTUP_TOOLCHAIN=stable QISKIT_BUILD_WITH_MIMALLOC=1"
 
 [tool.cibuildwheel.windows]
 test-command = "cp -r {project}/test . && stestr --test-path test/python run --abbreviate"

releasenotes/notes/mimalloc-403d3300aa698fae.yaml

@@ -0,0 +1,17 @@
+---
+features_misc:
+  - Qiskit can now optionally be built with `mimalloc <https://github.com/microsoft/mimalloc>`__ as the global
+    allocator. This improves the runtime performance of Qiskit and also should
+    reduce the resident set size when running Qiskit. By default the precompiled
+    wheels on PyPI will are built with mimalloc, when building from source you
+    need to opt-in and enable using mimalloc. Enabling mimalloc when building
+    from source will require a C compiler to be installed as mimalloc will be
+    built from source.
+build:
+  - A new environment variable, ``QISKIT_BUILD_WITH_MIMALLOC`` is used to
+    control whether to build Qiskit using mimalloc as the global allocator.
+    When ``QISKIT_BUILD_WITH_MIMALLOC=1`` this will enable using mimalloc for
+    both building the Python package and using the Makefile to build the
+    C API as a standalone shared library with ``make c``. If enabled building
+    Qiskit from source a C compiler is required to build
+    `mimalloc <https://github.com/microsoft/mimalloc>`__.

setup.py

@@ -131,9 +131,12 @@ def get_package_dir(package: str) -> Path:
 # is all in rust (default to no caching and make caching an opt-in feature). This is opt-out
 # right now to avoid the runtime overhead until we are leveraging the rust gates infrastructure.
 if os.getenv("QISKIT_NO_CACHE_GATES") == "1":
-    features = ["mimalloc"]
+    features = []
 else:
-    features = ["cache_pygates", "mimalloc"]
+    features = ["cache_pygates"]
+
+if os.getenv("QISKIT_BUILD_WITH_MIMALLOC") == "1":
+    features.append("mimalloc")
 
 
 setup(