Switch to heap types using PyType_FromSpec (#643)

* Switch to heap types using PyType_FromSpec

- Convert PyJuliaBase_Type creation from manual PyTypeObject to PyType_FromSpec
- Add PyType_Spec and PyType_Slot structs to C API bindings
- Add all Python type slot constants from typeslots.h
- Use __weaklistoffset__ member definition for older Python compatibility
- Implement proper slot-based type definition with buffer protocol support

Closes #641

* Clean up PyType_Spec creation

- Remove unnecessary itemsize field (defaults to 0)
- Remove unnecessary Cint conversion for basicsize (happens automatically)

* update release notes

* Remove unused _pyjlbase_as_buffer constant

Buffer protocol methods are now handled through PyType_Slot mechanism,
making the old _pyjlbase_as_buffer constant unnecessary.

* revert

---------

Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: cjdoris

docs/src/releasenotes.md

@@ -1,5 +1,8 @@
 # Release Notes
 
+## Unreleased
+* Internal: Use heap-allocated types (PyType_FromSpec) to improve ABI compatibility.
+
 ## 0.9.26 (2025-07-15)
 * Added PySide6 support to the GUI compatibility layer.
 * Added FAQ on interactive threads.

src/C/consts.jl

@@ -345,3 +345,104 @@ const NPY_ARRAY_ALIGNED = 0x0100
 const NPY_ARRAY_NOTSWAPPED = 0x0200
 const NPY_ARRAY_WRITEABLE = 0x0400
 const NPY_ARR_HAS_DESCR = 0x0800
+
+# Python type slot constants
+# From https://github.com/python/cpython/blob/main/Include/typeslots.h
+const Py_bf_getbuffer = Cint(1)
+const Py_bf_releasebuffer = Cint(2)
+const Py_mp_ass_subscript = Cint(3)
+const Py_mp_length = Cint(4)
+const Py_mp_subscript = Cint(5)
+const Py_nb_absolute = Cint(6)
+const Py_nb_add = Cint(7)
+const Py_nb_and = Cint(8)
+const Py_nb_bool = Cint(9)
+const Py_nb_divmod = Cint(10)
+const Py_nb_float = Cint(11)
+const Py_nb_floor_divide = Cint(12)
+const Py_nb_index = Cint(13)
+const Py_nb_inplace_add = Cint(14)
+const Py_nb_inplace_and = Cint(15)
+const Py_nb_inplace_floor_divide = Cint(16)
+const Py_nb_inplace_lshift = Cint(17)
+const Py_nb_inplace_multiply = Cint(18)
+const Py_nb_inplace_or = Cint(19)
+const Py_nb_inplace_power = Cint(20)
+const Py_nb_inplace_remainder = Cint(21)
+const Py_nb_inplace_rshift = Cint(22)
+const Py_nb_inplace_subtract = Cint(23)
+const Py_nb_inplace_true_divide = Cint(24)
+const Py_nb_inplace_xor = Cint(25)
+const Py_nb_int = Cint(26)
+const Py_nb_invert = Cint(27)
+const Py_nb_lshift = Cint(28)
+const Py_nb_multiply = Cint(29)
+const Py_nb_negative = Cint(30)
+const Py_nb_or = Cint(31)
+const Py_nb_positive = Cint(32)
+const Py_nb_power = Cint(33)
+const Py_nb_remainder = Cint(34)
+const Py_nb_rshift = Cint(35)
+const Py_nb_subtract = Cint(36)
+const Py_nb_true_divide = Cint(37)
+const Py_nb_xor = Cint(38)
+const Py_sq_ass_item = Cint(39)
+const Py_sq_concat = Cint(40)
+const Py_sq_contains = Cint(41)
+const Py_sq_inplace_concat = Cint(42)
+const Py_sq_inplace_repeat = Cint(43)
+const Py_sq_item = Cint(44)
+const Py_sq_length = Cint(45)
+const Py_sq_repeat = Cint(46)
+const Py_tp_alloc = Cint(47)
+const Py_tp_base = Cint(48)
+const Py_tp_bases = Cint(49)
+const Py_tp_call = Cint(50)
+const Py_tp_clear = Cint(51)
+const Py_tp_dealloc = Cint(52)
+const Py_tp_del = Cint(53)
+const Py_tp_descr_get = Cint(54)
+const Py_tp_descr_set = Cint(55)
+const Py_tp_doc = Cint(56)
+const Py_tp_getattr = Cint(57)
+const Py_tp_getattro = Cint(58)
+const Py_tp_hash = Cint(59)
+const Py_tp_init = Cint(60)
+const Py_tp_is_gc = Cint(61)
+const Py_tp_iter = Cint(62)
+const Py_tp_iternext = Cint(63)
+const Py_tp_methods = Cint(64)
+const Py_tp_new = Cint(65)
+const Py_tp_repr = Cint(66)
+const Py_tp_richcompare = Cint(67)
+const Py_tp_setattr = Cint(68)
+const Py_tp_setattro = Cint(69)
+const Py_tp_str = Cint(70)
+const Py_tp_traverse = Cint(71)
+const Py_tp_members = Cint(72)
+const Py_tp_getset = Cint(73)
+const Py_tp_free = Cint(74)
+const Py_nb_matrix_multiply = Cint(75)
+const Py_nb_inplace_matrix_multiply = Cint(76)
+const Py_am_await = Cint(77)
+const Py_am_aiter = Cint(78)
+const Py_am_anext = Cint(79)
+const Py_tp_finalize = Cint(80)
+const Py_am_send = Cint(81)
+const Py_tp_vectorcall = Cint(82)
+const Py_tp_token = Cint(83)
+
+# PyType_Spec and PyType_Slot structs
+# From https://docs.python.org/3/c-api/type.html#c.PyType_Spec
+@kwdef struct PyType_Slot
+    slot::Cint = 0
+    pfunc::Ptr{Cvoid} = C_NULL
+end
+
+@kwdef struct PyType_Spec
+    name::Cstring = C_NULL
+    basicsize::Cint = 0
+    itemsize::Cint = 0
+    flags::Cuint = 0
+    slots::Ptr{PyType_Slot} = C_NULL
+end

src/C/pointers.jl

@@ -77,6 +77,7 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyType_IsSubtype => (PyPtr, PyPtr) => Cint,
     :PyType_Ready => (PyPtr,) => Cint,
     :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr,
+    :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr,
     # MAPPING
     :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint,
     :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint,

src/JlWrap/C.jl

@@ -269,14 +269,16 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr)
 end
 
 const _pyjlbase_name = "juliacall.ValueBase"
-const _pyjlbase_type = fill(C.PyTypeObject())
 const _pyjlbase_isnull_name = "_jl_isnull"
 const _pyjlbase_callmethod_name = "_jl_callmethod"
 const _pyjlbase_reduce_name = "__reduce__"
 const _pyjlbase_serialize_name = "_jl_serialize"
 const _pyjlbase_deserialize_name = "_jl_deserialize"
+const _pyjlbase_weaklistoffset_name = "__weaklistoffset__"
 const _pyjlbase_methods = Vector{C.PyMethodDef}()
-const _pyjlbase_as_buffer = fill(C.PyBufferProcs())
+const _pyjlbase_members = Vector{C.PyMemberDef}()
+const _pyjlbase_slots = Vector{C.PyType_Slot}()
+const _pyjlbase_spec = fill(C.PyType_Spec())
 
 function init_c()
     empty!(_pyjlbase_methods)
@@ -309,25 +311,44 @@ function init_c()
         ),
         C.PyMethodDef(),
     )
-    _pyjlbase_as_buffer[] = C.PyBufferProcs(
-        get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)),
-        release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})),
+    
+    # Create members for weakref support
+    empty!(_pyjlbase_members)
+    push!(
+        _pyjlbase_members,
+        C.PyMemberDef(
+            name = pointer(_pyjlbase_weaklistoffset_name),
+            typ = C.Py_T_PYSSIZET,
+            offset = fieldoffset(PyJuliaValueObject, 3),
+            flags = C.Py_READONLY,
+        ),
+        C.PyMemberDef(), # NULL terminator
+    )
+    
+    # Create slots for PyType_Spec
+    empty!(_pyjlbase_slots)
+    push!(
+        _pyjlbase_slots,
+        C.PyType_Slot(slot = C.Py_tp_new, pfunc = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr))),
+        C.PyType_Slot(slot = C.Py_tp_dealloc, pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,))),
+        C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(_pyjlbase_methods)),
+        C.PyType_Slot(slot = C.Py_tp_members, pfunc = pointer(_pyjlbase_members)),
+        C.PyType_Slot(slot = C.Py_bf_getbuffer, pfunc = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint))),
+        C.PyType_Slot(slot = C.Py_bf_releasebuffer, pfunc = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer}))),
+        C.PyType_Slot(), # NULL terminator
     )
-    _pyjlbase_type[] = C.PyTypeObject(
+    
+    # Create PyType_Spec
+    _pyjlbase_spec[] = C.PyType_Spec(
         name = pointer(_pyjlbase_name),
         basicsize = sizeof(PyJuliaValueObject),
-        # new = C.POINTERS.PyType_GenericNew,
-        new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)),
-        dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)),
         flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG,
-        weaklistoffset = fieldoffset(PyJuliaValueObject, 3),
-        # getattro = C.POINTERS.PyObject_GenericGetAttr,
-        # setattro = C.POINTERS.PyObject_GenericSetAttr,
-        methods = pointer(_pyjlbase_methods),
-        as_buffer = pointer(_pyjlbase_as_buffer),
+        slots = pointer(_pyjlbase_slots),
     )
-    o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type))
-    if C.PyType_Ready(o) == -1
+    
+    # Create type using PyType_FromSpec
+    o = PyJuliaBase_Type[] = C.PyType_FromSpec(pointer(_pyjlbase_spec))
+    if o == C.PyNULL
         C.PyErr_Print()
         error("Error initializing 'juliacall.ValueBase'")
     end