185 official device simulations

benchmarks/fake_backend_5Q_benchmarks.toml

@@ -0,0 +1,66 @@
+# Version of the specification language/format, to manage future changes
+spec_version = "1.0"
+# Version of this particular benchmark suite, to manage any important changes
+suite_version = "0.1"
+# Unique identifier for this benchmark suite
+id = "fake_backend_5Q_benchmarks"
+# Human readable description
+description = "Compiler benchmarks running in small quantum device emulators."
+
+# --------- Compilers ----------
+# For now, restricting to qiskit derived compilers to work with the same target-device options
+[[compilers]]
+id = "ucc"
+[[compilers]]
+id = "qiskit-default"
+
+# --------- Target Devices ----------
+[[target_devices]]
+# 5 qubits
+id = "ibm_fake_manila"
+
+# --------- Benchmarks ----------
+
+[[benchmarks]]
+id = "mqt:qaoa_N5"
+description = "mqt:qaoa circuit (N=5)"
+simulate.measurement = "qaoa"
+generator.name = "mqt:qaoa"
+generator.params.N = 5
+
+
+
+[[benchmarks]]
+id = "qft_N5"
+description = "qft circuit (N=5)"
+simulate.measurement = "uniform_superposition_projector"
+generator.name = "qft"
+generator.params.N = 5
+
+
+
+[[benchmarks]]
+id = "prep_select_N5"
+description = "prep_select circuit (N=5)"
+simulate.measurement = "prep_select_all_ones"
+generator.name = "prep_select"
+generator.params.N = 5
+
+
+
+[[benchmarks]]
+id = "qcnn_N5"
+description = "qcnn circuit (N=5)"
+simulate.measurement = "qcnn"
+generator.name = "qcnn"
+generator.params.N = 5
+
+
+
+[[benchmarks]]
+id = "qv_N5"
+description = "qv circuit (N=5)"
+simulate.measurement = "heavy_output"
+generator.name = "qv"
+generator.params.N = 5
+

benchmarks/fake_backend_7Q_benchmarks.toml

@@ -0,0 +1,62 @@
+# Version of the specification language/format, to manage future changes
+spec_version = "1.0"
+# Version of this particular benchmark suite, to manage any important changes
+suite_version = "0.1"
+# Unique identifier for this benchmark suite
+id = "fake_backend_7Q_benchmarks"
+# Human readable description
+description = "Compiler benchmarks running in small quantum device emulators."
+
+# --------- Compilers ----------
+# For now, restricting to qiskit derived compilers to work with the same target-device options
+[[compilers]]
+id = "ucc"
+[[compilers]]
+id = "qiskit-default"
+
+# --------- Target Devices ----------
+[[target_devices]]
+# 7 qubits
+id = "ibm_fake_jakarta"
+
+
+# --------- Benchmarks ----------
+# Change all these to N=7 versions of the existing 5Q benchmarks
+
+# --------- Benchmarks ----------
+[[benchmarks]]
+id = "mqt:qaoa_N7"
+description = "mqt:qaoa circuit (N=7)"
+simulate.measurement = "qaoa"
+generator.name = "mqt:qaoa"
+generator.params.N = 7
+
+[[benchmarks]]
+id = "prep_select_N7"
+description = "prep_select circuit (N=7)"
+simulate.measurement = "prep_select_all_ones"
+generator.name = "prep_select"
+generator.params.N = 7
+
+[[benchmarks]]
+id = "qcnn_N7"
+description = "qcnn circuit (N=7)"
+simulate.measurement = "qcnn"
+generator.name = "qcnn"
+generator.params.N = 7
+
+[[benchmarks]]
+id = "qft_N7"
+description = "qft circuit (N=7)"
+simulate.measurement = "uniform_superposition_projector"
+generator.name = "qft"
+generator.params.N = 7
+
+[[benchmarks]]
+id = "qv_N7"
+description = "qv circuit (N=7)"
+simulate.measurement = "heavy_output"
+generator.name = "qv"
+generator.params.N = 7
+
+

plotting/plot_latest_benchmark.py

@@ -139,9 +139,16 @@ def main():
 
     # --- Plot Compilation Benchmarks ---
     if args.plot in ["all", "compilation"]:
-        db = SuiteResultsDatabase.from_root(
-            args.root_dir, args.runner_name, "compilation_benchmarks"
-        )
+        # If runner_name matches a known suite, use it directly
+        suite_dir = args.root_dir / args.runner_name
+        if (suite_dir / "compilation_benchmarks").exists():
+            db = SuiteResultsDatabase.from_root(
+                args.root_dir, args.runner_name, "compilation_benchmarks"
+            )
+        else:
+            db = SuiteResultsDatabase.from_root(
+                args.root_dir, args.runner_name, args.runner_name
+            )
 
         suite_results = db.from_uid(args.uid) if args.uid else db.get_latest()
         if suite_results is None:
@@ -164,18 +171,35 @@ def main():
 
     # --- Plot Simulation Benchmarks ---
     if args.plot in ["all", "simulation"]:
+        suite_dir = args.root_dir / args.runner_name
+        # Use the suite name as the keyword for database and CSV lookup
         db = SuiteResultsDatabase.from_root(
-            args.root_dir, args.runner_name, "simulation_benchmarks"
+            args.root_dir, args.runner_name, args.runner_name
         )
-
         suite_results = db.from_uid(args.uid) if args.uid else db.get_latest()
-        if suite_results is None:
-            print(f"No simulation data found for UID {args.uid}")
-            sys.exit(1)
-
-        latest_date = suite_results.metadata.uid_timestamp.strftime("%Y-%m-%d")
-
-        df = to_df_simulation(suite_results)
+        if suite_results is not None:
+            latest_date = suite_results.metadata.uid_timestamp.strftime("%Y-%m-%d")
+            df = to_df_simulation(suite_results)
+        else:
+            # Fallback: load latest simulation.csv directly
+            import glob
+            import os
+
+            csv_dir = str(suite_dir)
+            csv_files = glob.glob(
+                os.path.join(csv_dir, "**", "*.simulation.csv"), recursive=True
+            )
+            if not csv_files:
+                print(f"No simulation CSV files found in {csv_dir}")
+                sys.exit(1)
+            latest_csv = max(csv_files, key=os.path.getmtime)
+            print(f"Loading simulation data from {latest_csv}")
+            df = pd.read_csv(latest_csv)
+            # Try to extract date from filename or column
+            if "uid_timestamp" in df.columns:
+                latest_date = str(df["uid_timestamp"].iloc[0]).split()[0]
+            else:
+                latest_date = "unknown"
 
         file_ext = "pdf" if args.pdf else "png"
         out_path = (

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "seaborn>=0.13.2",
     "tomlkit>=0.13.3",
     "mqt-bench>=2.1.0",
+    "pre-commit>=4.4.0",
 ]
 
 [build-system]

src/ucc_bench/runner.py

@@ -5,6 +5,7 @@
 import logging
 from logging import LoggerAdapter
 from concurrent.futures import ProcessPoolExecutor
+
 from .suite import BenchmarkSuite, BenchmarkSpec
 from .compilers import BaseCompiler, DEFAULT_GATESET
 from .results import BenchmarkResult, CompilerInfo, CompilationMetrics
@@ -123,13 +124,23 @@ def process(self, msg, kwargs):
 
         if register.has_observable(benchmark.simulate.measurement):
             observable = register.get_observable(benchmark.simulate.measurement)
-            simulation_metrics = calc_expectation_value(
-                observable(raw_circuit_qiskit.num_qubits),
-                raw_circuit_qiskit,
-                compiled_circuit_qiskit,
-                simulator,
-            )
-            simulation_metrics.measurement_id = observable._id
+            # If an output metric is registered, use it
+            if register.has_output_metric(observable._id):
+                output_metric = register.get_output_metric(observable._id)
+                simulation_metrics = output_metric(
+                    raw_circuit_qiskit,
+                    compiled_circuit_qiskit,
+                    simulator,
+                )
+                simulation_metrics.measurement_id = observable._id
+            else:
+                simulation_metrics = calc_expectation_value(
+                    observable(raw_circuit_qiskit.num_qubits),
+                    raw_circuit_qiskit,
+                    compiled_circuit_qiskit,
+                    simulator,
+                )
+                simulation_metrics.measurement_id = observable._id
         elif register.has_output_metric(benchmark.simulate.measurement):
             output_metric = register.get_output_metric(benchmark.simulate.measurement)
             simulation_metrics = output_metric(

src/ucc_bench/simulation/observables.py

@@ -1,3 +1,60 @@
+from typing import Union
+from math import sqrt
+from qiskit import QuantumCircuit, ClassicalRegister
+from qiskit.quantum_info import Operator, Statevector, SparsePauliOp
+from qiskit_aer import AerSimulator
+import numpy as np
+from qiskit.result import Counts
+from ..registry import register
+from ..results import SimulationMetrics
+
+
+def calc_computational_basis_expectation(
+    uncompiled_circuit: QuantumCircuit,
+    compiled_circuit: QuantumCircuit,
+    simulator: AerSimulator,
+) -> SimulationMetrics:
+    # Ensure classical bits and measurement
+    def ensure_classical_bits_and_measurement(circuit):
+        num_qubits = circuit.num_qubits
+        if circuit.num_clbits < num_qubits:
+            cr = ClassicalRegister(num_qubits - circuit.num_clbits)
+            circuit.add_register(cr)
+        if not any(instr[0].name == "measure" for instr in circuit.data):
+            circuit.measure(range(num_qubits), range(num_qubits))
+
+    ensure_classical_bits_and_measurement(uncompiled_circuit)
+    ensure_classical_bits_and_measurement(compiled_circuit)
+
+    shots = 1024
+    ideal_result = simulator.run(uncompiled_circuit, shots=shots).result()
+    ideal_counts = ideal_result.get_counts()
+    noisy_result = simulator.run(compiled_circuit, shots=shots).result()
+    noisy_counts = noisy_result.get_counts()
+
+    def z_expectation(counts: Counts, num_qubits: int):
+        total = 0
+        shots = sum(counts.values())
+        for bitstring, count in counts.items():
+            val = 1 if bitstring.count("1") % 2 == 0 else -1
+            total += val * count
+        return total / shots if shots > 0 else 0.0
+
+    uncompiled_ideal = z_expectation(ideal_counts, uncompiled_circuit.num_qubits)
+    compiled_noisy = z_expectation(noisy_counts, compiled_circuit.num_qubits)
+
+    import math
+
+    return SimulationMetrics(
+        uncompiled_ideal=uncompiled_ideal,
+        compiled_ideal=math.nan,
+        uncompiled_noisy=math.nan,
+        compiled_noisy=compiled_noisy,
+    )
+
+
+register.output_metric("computational_basis")(calc_computational_basis_expectation)
+
 """
 This module provides functionality calculating expectation values of compiled circuits
 with and without noise. It has some common functions for calculating these values given an
@@ -8,15 +65,6 @@
 in the circuit as an argument and return a Qiskit Operator representing the observable to measure.
 """
 
-from typing import Union
-from math import sqrt
-from qiskit import QuantumCircuit
-from qiskit.quantum_info import Operator, Statevector, SparsePauliOp
-from qiskit_aer import AerSimulator
-import numpy as np
-from ..registry import register
-from ..results import SimulationMetrics
-
 # ----------------------------------------------------
 # Simulation functions to calculate expectation values
 # ----------------------------------------------------
@@ -151,14 +199,25 @@ def generate_square_heisenberg_observable(num_qubits):
 
 @register.observable("qaoa")
 def generate_qaoa_observable(num_qubits):
-    """Generates the problem Hamiltonian as the observable for the QAOA
-    benchmarking circuits, based on the binary encoding described in
-    Franz G. Fuchs, Herman Øie Kolden, Niels Henrik Aase, and Giorgio
-    Sartor "Efficient encoding of the weighted MAX k-CUT on a quantum computer
-    using QAOA". (2020) arXiv 2009.01095 (https://arxiv.org/abs/2009.01095).
-    The weights of the edges between vertices and of the resulting unitary
-    evolution come from the 10-vertex Barabasi-Albert graph in Fig 4(c)
-    of the paper.
+    """Generate the problem Hamiltonian observable for QAOA benchmarks.
+
+    Notes
+    -----
+    The original reference graph (Fuchs et al. 2020, arXiv:2009.01095) uses a
+    10-vertex Barabasi-Albert instance. The hard-coded edge list below encodes
+    that graph. Some benchmark suites in this project request QAOA circuits
+    with fewer than 10 qubits (e.g. N=5 or N=7). In those cases, the original
+    edge list contains vertex indices that exceed ``num_qubits - 1`` which
+    previously caused an ``IndexError: list assignment index out of range``.
+
+    To make the observable generation work for ``num_qubits < 10`` we filter
+    out edges that touch vertices outside the available range.
+
+    If fewer than 2 valid vertices remain after filtering (i.e. ``num_qubits < 2``)
+    we raise a ValueError because no meaningful 2-local ZZ Hamiltonian can be
+    constructed.
+        TODO: In general, we should define an observable for QAOA which allows an arbitrary number of qubits and generates the corresponding graph structure.
+
     """
     pauli_strings = []
     # Weights of edges between vertices and of the resulting unitary evolution
@@ -188,15 +247,20 @@ def generate_qaoa_observable(num_qubits):
         (7, 9, 4.265),
         (8, 9, 1.690),
     ]
-    for i, j, _ in weighted_edges:
-        # Start with identity string
+    if num_qubits < 2:
+        raise ValueError(
+            f"QAOA observable requires at least 2 qubits; got num_qubits={num_qubits}"
+        )
+    # Filter edges to those within the available qubit index range
+    filtered_edges = [
+        (i, j, w) for (i, j, w) in weighted_edges if i < num_qubits and j < num_qubits
+    ]
+    for i, j, weight in filtered_edges:
         pauli_string = ["I"] * num_qubits
-        # Place Z operators on the chosen qubits
         pauli_string[i] = "Z"
         pauli_string[j] = "Z"
-        # Convert to PauliSumOp
         pauli_strings.append("".join(pauli_string))
-    coeffs = [weight for _, _, weight in weighted_edges]
+    coeffs = [weight for _, _, weight in filtered_edges]
     observable = SparsePauliOp(pauli_strings, coeffs)
     return observable