Add safe Python-extension module C API headers (#15762)

* Add safe Python-extension module C API headers

With the vtables now compiled into the `_accelerate` object and stored
in suitable `PyCapsule`s, the last step to exposing the complete ability
to compile Python extension modules is providing header-file support for
actually using the result.  This support is modelled on NumPy.

We generate alternate versions of the function declarations as part of
the `pyext` build script, which are loaded (instead of the standard
function prototypes) when the `QISKIT_PYTHON_EXTENSION` macro is set
prior to the inclusion of `qiskit.h`.  These declarations are all
pre-processor macros that resolve to compile-time constant offset
lookups into the vtables stored in the `PyCapsule`s, except we cache the
internal pointer of each `PyCapsule` into a compilation-unit-local
`static`.  If we didn't have this cache, _all_ function calls would have
Python-API overhead and require an attached Python thread state (holding
the GIL).

The cache population is done by a new header-only function `qk_import`
defined in (the non-stub version of) `funcs_py.h`, which then must be
called _before_ any C API function.  This will almost invariably be done
inside the `PyInit_*` module-initialisation function of the extension.

The cache mechanism introduced in this commit is local to a single
translation unit.  It is possible to extend this to allow sharing it
between different translation units, but since this necessarily requires
exposing a non-`static` symbol out of a library, we will have to take
care to do it with a mechanism that allows the user to override the
names used.

* Remove `dbg!` call from build script

* Remove unneeded `pyo3-build-config` dependence

* Fix typos

Co-authored-by: Max Rossmannek <21973473+mrossinek@users.noreply.github.com>
Co-authored-by: Jake Lishman <jake@binhbar.com>

---------

Co-authored-by: Max Rossmannek <21973473+mrossinek@users.noreply.github.com>

Author: jakelishman

Cargo.lock

@@ -69,15 +69,6 @@ pub static FN_DEPRECATED_WITH_NOTE: &str = "Qk_DEPRECATED_FN_NOTE({})";
 pub static CFG_FEATURE_DEFINES: &[(&str, &str)] =
     &[(PYTHON_BINDING_FEATURE, PYTHON_BINDING_DEFINE)];
 
-fn guarded_python_import(guard: &str) -> String {
-    format!(
-        "\
-#ifdef {guard}
-#include <Python.h>
-#endif"
-    )
-}
-
 #[inline]
 fn to_vec_string(slice: &[&str]) -> Vec<String> {
     slice.iter().map(|s| String::from(*s)).collect()
@@ -117,14 +108,6 @@ fn manual_include_files() -> anyhow::Result<Vec<PathBuf>> {
 
 /// Get the Qiskit configuration
 fn get_config() -> anyhow::Result<cbindgen::Config> {
-    // `Python.h` is required to be the first file included because it reserves the right to define
-    // preprocessor macros that affect standard-library includes.  This causes it to be ahead of our
-    // include guard, but `Python.h` has its own, so we should be fine.
-    let header = Some(format!(
-        "{}\n{}",
-        COPYRIGHT,
-        guarded_python_import(PYTHON_BINDING_DEFINE)
-    ));
     // We need to include the `attributes.h` file in all generated files to make sure Doxygen can
     // understand the deprecated attributes (even though `qiskit.h` is organised to include it).
     let includes = vec![
@@ -173,7 +156,7 @@ fn get_config() -> anyhow::Result<cbindgen::Config> {
         .map(|&(cfg, def)| (format!("feature = {cfg}"), String::from(def)))
         .collect();
     Ok(cbindgen::Config {
-        header,
+        header: Some(COPYRIGHT.to_owned()),
         language: cbindgen::Language::C,
         includes,
         include_version: true,

crates/bindgen/src/lib.rs

@@ -20,12 +20,14 @@ workspace = true
 # `libpython` is *not* linked, and if the feature isn't present, then it is.  To test the Rust
 # crates as standalone binaries, executables, we need `libpython` to be linked in, so we make the
 # feature a default, and run `cargo test --no-default-features` to turn it off.
-default = ["pyo3/extension-module"]
-cache_pygates = ["pyo3/extension-module", "qiskit-circuit/cache_pygates", "qiskit-accelerate/cache_pygates", "qiskit-transpiler/cache_pygates", "qiskit-cext/cache_pygates", "qiskit-qpy/cache_pygates"]
+cache_pygates = ["qiskit-circuit/cache_pygates", "qiskit-accelerate/cache_pygates", "qiskit-transpiler/cache_pygates", "qiskit-cext/cache_pygates", "qiskit-qpy/cache_pygates"]
 
 [build-dependencies]
 anyhow.workspace = true
+cbindgen = { workspace = true, features = ["unstable_ir"] }
+hashbrown.workspace = true
 qiskit-bindgen.workspace = true
+qiskit-cext-vtable = { workspace = true, features = ["python_binding"] }
 
 [dependencies]
 pyo3.workspace = true

crates/pyext/Cargo.toml

@@ -10,23 +10,147 @@
 // copyright notice, and modified files need to carry a notice indicating
 // that they have been altered from the originals.
 
-use anyhow::anyhow;
+use cbindgen::bindgen::ir;
+use hashbrown::HashMap;
+use qiskit_cext_vtable::{FUNCTIONS_CIRCUIT, FUNCTIONS_QI, FUNCTIONS_TRANSPILE};
+use std::fs;
+use std::io::Write;
 use std::path::Path;
 
+static WRAPPER_FUNCS: &str = "funcs_py.h";
+static GENERATED_FUNCS: &str = "funcs_py_generated.h";
+
+/// Render a given type object into a string representing it in C.
+fn render_type_as_c(ty: &ir::Type, config: &cbindgen::Config) -> String {
+    fn render(ty: &ir::Type, config: &cbindgen::Config, acc: &mut String) {
+        match ty {
+            ir::Type::Ptr {
+                ty,
+                is_const,
+                is_nullable: _,
+                is_ref,
+            } => {
+                assert!(!is_ref, "C++ reference-likes not handled");
+                if *is_const {
+                    acc.push_str("const ");
+                }
+                render(ty, config, acc);
+                acc.push_str(" *");
+            }
+            ir::Type::Path(p) => acc.push_str(p.export_name()),
+            ir::Type::Primitive(ty) => acc.push_str(ty.to_repr_c(config)),
+            ir::Type::Array(..) => todo!("array types not yet handled"),
+            ir::Type::FuncPtr {
+                args,
+                ret,
+                is_nullable,
+                never_return,
+            } => {
+                assert!(!is_nullable, "nullability of funcptrs is not handled");
+                assert!(!never_return, "diverging functions not handled");
+                render(ret, config, acc);
+                acc.push_str("(*)(");
+                let mut args = args.iter();
+                if let Some((_, first)) = args.next() {
+                    render(first, config, acc);
+                    for (_, arg) in args {
+                        acc.push_str(", ");
+                        render(arg, config, acc)
+                    }
+                }
+                acc.push(')');
+            }
+        }
+    }
+    let mut acc = String::new();
+    render(ty, config, &mut acc);
+    acc
+}
+
+/// Calculate a mapping of exported function names to C casts to appropriate function-pointer types.
+fn functions_as_c_funcptr_casts(bindings: &cbindgen::Bindings) -> HashMap<&str, String> {
+    let to_funcptr = |func: &ir::Function| {
+        let to_funcptr_arg = |arg: &ir::FunctionArgument| {
+            let ir::FunctionArgument {
+                name: _,
+                ty,
+                array_length,
+            } = arg;
+            assert!(array_length.is_none(), "array arguments not handled");
+            (None, ty.clone())
+        };
+        ir::Type::FuncPtr {
+            ret: Box::new(func.ret.clone()),
+            args: func.args.iter().map(to_funcptr_arg).collect(),
+            is_nullable: false,
+            never_return: false,
+        }
+    };
+    let config = &bindings.config;
+    bindings
+        .functions
+        .iter()
+        .map(|func| {
+            let funcptr = to_funcptr(func);
+            (func.path.name(), render_type_as_c(&funcptr, config))
+        })
+        .collect()
+}
+
+/// Install (overwriting) the Python-extension-specific header files into the given directory.
+fn install_py_function_headers(
+    bindings: &cbindgen::Bindings,
+    install_path: impl AsRef<Path>,
+) -> anyhow::Result<()> {
+    let mut our_include = Path::new(env!("CARGO_MANIFEST_DIR")).join("include");
+    our_include.push(qiskit_bindgen::SCOPED_INCLUDE_DIR);
+    // This directory must already have been constructed by the previous "install" command for the
+    // regular C headers; if it doesn't, writing out the files will be a mistake because we _should_
+    // be overwriting an existing file (the wrapper that defines `qk_import`).
+    let install_path = install_path
+        .as_ref()
+        .join(qiskit_bindgen::SCOPED_INCLUDE_DIR);
+    fs::copy(
+        our_include.join(WRAPPER_FUNCS),
+        install_path.join(WRAPPER_FUNCS),
+    )?;
+    let mut funcs_header = fs::File::create(install_path.join(GENERATED_FUNCS))?;
+    writeln!(funcs_header, "{}", qiskit_bindgen::COPYRIGHT)?;
+
+    // Now, each function's name is just a preprocessor macro that resolves to a lookup into the
+    // corresponding table.  The names given here need to match with the handwritten include file
+    // that sets up the slots in `qk_import`.
+    let vtables = [
+        ("_Qk_API_Circuit", &FUNCTIONS_CIRCUIT),
+        ("_Qk_API_Transpile", &FUNCTIONS_TRANSPILE),
+        ("_Qk_API_QI", &FUNCTIONS_QI),
+    ];
+    let funcs = functions_as_c_funcptr_casts(bindings);
+    for (vtable_name, vtable) in vtables {
+        for export in vtable.exports(0) {
+            writeln!(
+                funcs_header,
+                "#define {} (*({})({}[{}]))",
+                export.name, funcs[export.name], vtable_name, export.slot
+            )?;
+        }
+    }
+    Ok(())
+}
+
 #[allow(clippy::print_stdout)] // We're a build script - we're _supposed_ to print to stdout.
 fn main() -> anyhow::Result<()> {
+    // Our actual requirements for re-running the build script are if `cext-vtable` changes, but
+    // since that's a build-time dependency, it's already implicit in Cargo's logic, so we just need
+    // to issue _any_ re-run command to avoid the default behaviour of rerunning if `qiskit_pyext`
+    // itself changes.
+    println!("cargo::rerun-if-changed=build.rs");
     let cext_path = {
         let mut path = Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf();
         path.pop();
         path.push("cext");
         path
     };
-    println!(
-        "cargo::rerun-if-changed={}",
-        cext_path
-            .to_str()
-            .ok_or_else(|| anyhow!("cext path isn't unicode"))?
-    );
     let out_path = {
         let out_dir = std::env::var("OUT_DIR").expect("cargo should set this for build scripts");
         let mut path = Path::new(&out_dir).to_path_buf();
@@ -37,5 +161,6 @@ fn main() -> anyhow::Result<()> {
     // We install the headers into our `OUT_DIR`, then we configure `setuptools-rust` to pick them
     // up from there and put them into the Python package.
     qiskit_bindgen::install_c_headers(&mut bindings, &out_path)?;
+    install_py_function_headers(&bindings, &out_path)?;
     Ok(())
 }

crates/pyext/build.rs

@@ -0,0 +1,59 @@
+// This code is part of Qiskit.
+//
+// (C) Copyright IBM 2026
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE.txt file in the root directory
+// of this source tree or at https://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+#if !defined(QISKIT_FUNCS_PY_H)
+#define QISKIT_FUNCS_PY_H
+
+// We rely on `qiskit.h` to `#include <Python.h>` on our behalf, since it needs to be included
+// before all other includes. (Though really, a user should also have included it themselves,
+// surely, if they're declaring a Python extension module.)
+
+// Declaring these as `static` in the header file means that there is a separate copy per
+// compilation unit.  This means that compiled extension modules written in C and using this need
+// to have all their Qiskit C API calls in the same file as the module definition.  There are ways
+// around this we can add in the future.
+static void **_Qk_API_Circuit;
+static void **_Qk_API_Transpile;
+static void **_Qk_API_QI;
+
+/**
+ * Import the Qiskit C API.
+ *
+ * @return 0 on success, or negative on failure.  If a failure occurs, the Python exception state
+ *     will be set.
+ *
+ * This function must be called before any attempt to use the Qiskit C API within this translation
+ * unit.  You must be attached to a Python interpreter to call this function.
+ */
+static int qk_import(void) {
+    PyObject *accelerate = PyImport_ImportModule("qiskit._accelerate");
+    if (!accelerate)
+        return -1;
+    // We don't actually need a handle to `accelerate` ourselves, we just need to have ensured it's
+    // already been imported.
+    Py_DECREF(accelerate);
+
+    _Qk_API_Circuit = (void **)PyCapsule_Import("qiskit._accelerate.capi.QK_FFI_CIRCUIT", 0);
+    if (!_Qk_API_Circuit)
+        return -1;
+    _Qk_API_Transpile = (void **)PyCapsule_Import("qiskit._accelerate.capi.QK_FFI_TRANSPILE", 0);
+    if (!_Qk_API_Transpile)
+        return -1;
+    _Qk_API_QI = (void **)PyCapsule_Import("qiskit._accelerate.capi.QK_FFI_QI", 0);
+    if (!_Qk_API_QI)
+        return -1;
+    return 0;
+}
+
+#include "qiskit/funcs_py_generated.h"
+
+#endif // QISKIT_FUNCS_PY_H

crates/pyext/include/qiskit/funcs_py.h

@@ -0,0 +1,46 @@
+=============
+Configuration
+=============
+
+The Qiskit C API has minimal configuration.
+
+.. c:macro:: QISKIT_PYTHON_EXTENSION
+
+   If defined before including ``qiskit.h``, the header files will define all of symbols in a mode
+   safe for use with Python extension modules.  This means that all API functions will evaluate to
+   function-pointer dereferences from a lookup table.
+
+   This is a user-defined macro and not defined by Qiskit itself.
+
+.. c:function:: int qk_import(void)
+
+   Import the Qiskit C API from the Python-space :py:mod:`qiskit` package.
+
+   You must call this once per compilation unit, before attempting to call any C API functions.
+   Failure to do so will typically result in null-pointer dereferences at runtime.
+
+   You typically will want to do this inside your ``PyInit_*`` module initialization function.  For
+   example, in ``my_extension.c``:
+
+   .. code-block:: c
+
+      #define QISKIT_PYTHON_EXTENSION
+      #include <Python.h>
+      #include <qiskit.h>
+
+      static struct PyModuleDef my_extension_mod = {
+         .m_base = PyModuleDef_HEAD_INIT,
+         .m_name = "my_extension",
+      };
+
+      PyMODINIT_FUNC PyInit_my_extension(void) {
+         if (qk_import() < 0) {
+            return NULL;
+         }
+         return PyModuleDef_Init(&my_extension_mod);
+      }
+
+   This function is only defined when :c:macro:`QISKIT_PYTHON_EXTENSION` was defined prior to
+   including ``qiskit.h``.
+
+   :return: 0 on success, negative on failure.

docs/cdoc/config.rst

@@ -88,4 +88,5 @@ Utilities
 .. toctree::
    :maxdepth: 1
 
+   config
    version

docs/cdoc/index.rst

@@ -0,0 +1,24 @@
+---
+features_c:
+  - |
+    Qiskit now supports building Python extension modules that use Qiskit C API from a version of
+    Qiskit loaded by Python.  These extension modules can then be safely distributed in wheels.
+
+    To compile an extension module, you must:
+
+    * define the :c:macro:`QISKIT_PYTHON_EXTENSION` macro before ``#include <qiskit.h>``.
+    * call :c:func:`qk_import` before attempting to use any Qiskit C API functions, once per
+      compilation unit (typically a single ``.c`` file).  Typically this will be in your extension
+      module's ``PyInit_*`` function.
+
+    ..
+      TODO: before the final release of 2.4, insert links to documentation/guides about this.
+
+
+    We intend to make it possible to have multiple compilation units using the Qiskit C API in an
+    extension module in a later release of Qiskit.
+
+    Beware that Qiskit's C API is still unstable; extension modules compiled against Qiskit 2.4 are
+    not guaranteed to be able to run against Qiskit 2.5 or later.  Qiskit will commit to this
+    stability at a later date.
+