fix deadlock when initializing enum via into_pyobject() (#5928)

* fix deadlock when initializing enum via `into_pyobject()`

* newsfragment

* fix `unreachable_code` failure

Author: davidhewitt

newsfragments/5928.fixed.md

@@ -0,0 +1 @@
+Fix deadlock in `.into_pyobject()` implementation for C-like `#[pyclass]` enums.

pyo3-macros-backend/src/pyclass.rs

@@ -1143,10 +1143,15 @@ fn impl_simple_enum(
                             Self::#variant_idents => #i,
                         )*
                     };
-                    #[allow(unreachable_code)]
-                    SINGLETON[idx].get_or_try_init(py, || {
-                        #pyo3_path::Py::new(py, self)
-                    }).map(|obj| ::std::clone::Clone::clone(obj.bind(py)))
+                    #[allow(unreachable_code)] // May happen if all variants are disabled by cfg attributes
+                    {
+                        // Ensure that the type object is initialized before we attempt to initialize any of the singleton instances,
+                        // to avoid deadlocking.
+                        <Self as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object().get_or_try_init(py)?;
+                        SINGLETON[idx].get_or_try_init(py, || {
+                            #pyo3_path::Py::new(py, self)
+                        }).map(|obj| ::std::clone::Clone::clone(obj.bind(py)))
+                    }
                 }
             }
         }

tests/test_enum.rs

@@ -22,6 +22,26 @@ fn test_enum_class_attr() {
     })
 }
 
+#[test]
+fn test_enum_intopyobject() {
+    // Variant of the above that goes via `.into_pyobject()`
+    // - regression test for https://github.com/PyO3/pyo3/issues/5927
+    //
+    #[pyclass(eq, eq_int, from_py_object)]
+    #[derive(Debug, PartialEq, Eq, Clone)]
+    pub enum MyEnum {
+        A,
+    }
+
+    Python::attach(|py| {
+        let var = MyEnum::A.into_pyobject(py).unwrap();
+        // NB important to call this after `into_pyobject` - this ensures
+        // that `.into_pyobject()` has to lazily initialize the enum type object
+        let my_enum = py.get_type::<MyEnum>();
+        py_assert!(py, my_enum var, "my_enum.A == var");
+    });
+}
+
 #[test]
 fn test_enum_eq_enum() {
     Python::attach(|py| {