Fix: support CPython free-threaded builds (3.14t) (#746)

* fix: support CPython free-threaded builds

Co-authored-by: Miles Cranmer <[email protected]>

* ci: test with python3.14t

Co-authored-by: Miles Cranmer <[email protected]>

* refactor: centralize free-threaded layout branching

Co-authored-by: Miles Cranmer <[email protected]>

* fix: make @ft macro hygienic

* run freethreading tests as cases of existing jobs and include python tests

* skip pycall tests on free-threaded python

* simplify implementation of ft macro

* add a comment for later

* update changelog

* extra care to ensure pythoncall uses the correct python executable

---------

Co-authored-by: Miles Cranmer <[email protected]>
Co-authored-by: Christopher Rowley <github.com/cjdoris>

Author: MilesCranmerBot

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   julia:
-    name: Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }})
+    name: Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }} ${{ matrix.pyversion}})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -21,11 +21,18 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
         jlversion: ['1','1.10']
         pythonexe: ['@CondaPkg']
+        pyversion: ['3']
         include:
         - arch: x64
           os: ubuntu-latest
           jlversion: '1'
           pythonexe: python
+          pyversion: '3'
+        - arch: x64
+          os: ubuntu-latest
+          jlversion: '1'
+          pythonexe: python
+          pyversion: '3.14t'
 
     steps:
       - uses: actions/checkout@v6
@@ -43,20 +50,27 @@ jobs:
         env:
           PYTHON: python
 
+      - name: Install Python
+        id: setup-python
+        uses: actions/setup-python@v6
+        if: ${{ matrix.pythonexe == 'python' }}
+        with:
+          python-version: ${{ matrix.pyversion }}
+      
       - name: Build PyCall
         if: ${{ matrix.pythonexe == 'python' }}
         run: |
           julia --project=test -e 'import Pkg; Pkg.build("PyCall")'
         env:
-          PYTHON: python
+          PYTHON: ${{ steps.setup-python.outputs.python-path }}
 
       - name: Run tests
         uses: julia-actions/julia-runtest@v1
         env:
           JULIA_DEBUG: PythonCall
           JULIA_NUM_THREADS: '2'
-          PYTHON: python
-          JULIA_PYTHONCALL_EXE: ${{ matrix.pythonexe }}
+          PYTHON: ${{ steps.setup-python.outputs.python-path }}
+          JULIA_PYTHONCALL_EXE: ${{ case(matrix.pythonexe == 'python', steps.setup-python.outputs.python-path, matrix.pythonexe) }}
 
       - name: Process coverage
         uses: julia-actions/julia-processcoverage@v1
@@ -79,6 +93,9 @@ jobs:
         - os: ubuntu-latest
           pyversion: '3'
           juliaexe: julia
+        - os: ubuntu-latest
+          pyversion: '3.14t'
+          juliaexe: '@JuliaPkg'
     env:
       MANUAL_TEST_PROJECT: /tmp/juliacall-test-project
       PYTHON_JULIACALL_THREADS: '2'

CHANGELOG.md

@@ -5,6 +5,7 @@
   to the corresponding NumPy dtype, like `numpy.dtype(jl.Int)`.
 * JuliaCall now launches Julia with 1 thread by default.
 * Added options `trace_compile` and `trace_compile_timing` to JuliaCall.
+* Initial experimental support for free-threaded Python 3.14.
 * Bug fixes.
 
 ## 0.9.31 (2025-12-17)

src/C/consts.jl

@@ -124,6 +124,20 @@ end
     type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved
 end
 
+@kwdef struct PyMutex
+    bits::Cuchar = 0
+end
+
+@kwdef struct PyObjectFT
+    tid::Csize_t = 0
+    flags::Cushort = 0
+    mutex::PyMutex = PyMutex()
+    gc_bits::Cuchar = 0
+    ref_local::Cuint = 0
+    ref_shared::Py_ssize_t = 0
+    type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved
+end
+
 const PyPtr = Ptr{PyObject}
 const PyNULL = PyPtr(0)
 
@@ -139,6 +153,11 @@ Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr
     size::Py_ssize_t = 0
 end
 
+@kwdef struct PyVarObjectFT
+    ob_base::PyObjectFT = PyObjectFT()
+    size::Py_ssize_t = 0
+end
+
 @kwdef struct PyMethodDef
     name::Cstring = C_NULL
     meth::Ptr{Cvoid} = C_NULL
@@ -249,6 +268,16 @@ end
     weakreflist::PyPtr = PyNULL
 end
 
+@kwdef struct PyMemoryViewObjectFT
+    ob_base::PyVarObjectFT = PyVarObjectFT()
+    mbuf::PyPtr = PyNULL
+    hash::Py_hash_t = 0
+    flags::Cint = 0
+    exports::Py_ssize_t = 0
+    view::Py_buffer = Py_buffer()
+    weakreflist::PyPtr = PyNULL
+end
+
 @kwdef struct PyTypeObject
     ob_base::PyVarObject = PyVarObject()
     name::Cstring = C_NULL
@@ -327,6 +356,11 @@ end
     value::T
 end
 
+@kwdef struct PySimpleObjectFT{T}
+    ob_base::PyObjectFT = PyObjectFT()
+    value::T
+end
+
 @kwdef struct PyArrayInterface
     two::Cint = 0
     nd::Cint = 0

src/C/context.jl

@@ -17,6 +17,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers,
     pyhome_w::Any = missing
     which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown
     version::Union{VersionNumber,Missing} = missing
+    is_free_threaded::Bool = false
 end
 
 const CTX = Context()
@@ -312,10 +313,11 @@ function init_context()
     v"3.10" ≤ CTX.version < v"4" || error(
         "Only Python 3.10+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).",
     )
+    CTX.is_free_threaded = occursin("free-threading build", verstr)
 
     launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable
 
-    @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version
+    @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded
 
     return
 end

src/C/extras.jl

@@ -1,44 +1,88 @@
 asptr(x) = Base.unsafe_convert(PyPtr, x)
 
-Py_Type(x) = Base.GC.@preserve x PyPtr(UnsafePtr(asptr(x)).type[!])
+# Free-threaded CPython builds ("3.14t") currently have different C struct layouts,
+# but there is no stable ABI yet. To keep the code manageable, we centralize the
+# branching in a single macro that rewrites type names in the expression.
+const _FT_TYPE_REPLACEMENTS = Dict{Symbol,Symbol}(
+    :PyObject => :PyObjectFT,
+    :PyVarObject => :PyVarObjectFT,
+    :PyMemoryViewObject => :PyMemoryViewObjectFT,
+    :PySimpleObject => :PySimpleObjectFT,
+    # Used from JlWrap/C.jl via `C.@ft`.
+    :PyJuliaValueObject => :PyJuliaValueObjectFT,
+)
+
+function _ft_transform(ex)
+    if ex isa Symbol
+        return get(_FT_TYPE_REPLACEMENTS, ex, ex)
+    elseif ex isa QuoteNode
+        return QuoteNode(_ft_transform(ex.value))
+    elseif ex isa Expr
+        return Expr(ex.head, map(_ft_transform, ex.args)...)
+    else
+        return ex
+    end
+end
+
+"""
+    @ft expr
+
+Evaluate `expr`, but when `CTX.is_free_threaded` is true (CPython "free-threaded"
+builds), rewrite internal type names like `PyObject` → `PyObjectFT` inside the
+expression.
+
+This keeps free-threaded branching centralized, so we don't scatter `if
+CTX.is_free_threaded` throughout the code.
+"""
+macro ft(ex)
+    ex_ft = _ft_transform(ex)
+    m = @__MODULE__
+    return esc(:($m.CTX.is_free_threaded ? $ex_ft : $ex))
+end
+
+Py_Type(x) = Base.GC.@preserve x @ft PyPtr(UnsafePtr{PyObject}(asptr(x)).type[!])
 
 PyObject_Type(x) = Base.GC.@preserve x (t = Py_Type(asptr(x)); Py_IncRef(t); t)
 
 Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), asptr(t))
 Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f)
 
 PyType_IsSubtypeFast(t, f::Integer) =
-    Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f))
+    Base.GC.@preserve t Cint(!iszero(PyType_GetFlags(asptr(t)) & f))
 
-PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
+PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m @ft Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
 
 PyType_CheckBuffer(t) = Base.GC.@preserve t begin
-    p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[]
-    return p != C_NULL && p.get[!] != C_NULL
+    getbuf = PyType_GetSlot(asptr(t), Py_bf_getbuffer)
+    return getbuf != C_NULL
 end
 
 PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o)))
 
 PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin
     o = asptr(_o)
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if p == C_NULL || p.get[!] == C_NULL
-        PyErr_SetString(
-            POINTERS.PyExc_TypeError,
-            "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'",
-        )
+    getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer)
+    if getbuf == C_NULL
+        # TODO: we can drop this branch and just use PyType_GetName once we stop
+        # supporting python 3.10
+        msg = if CTX.is_free_threaded
+            "a bytes-like object is required"
+        else
+            "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'"
+        end
+        PyErr_SetString(POINTERS.PyExc_TypeError, msg)
         return Cint(-1)
     end
-    return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
+    return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
 end
 
 PyBuffer_Release(_b) = begin
     b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b))
     o = b.obj[]
     o == C_NULL && return
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if (p != C_NULL && p.release[!] != C_NULL)
-        ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
+    releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer)
+    if releasebuf != C_NULL
+        ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
     end
     b.obj[] = C_NULL
     Py_DecRef(o)
@@ -65,7 +109,7 @@ function PyOS_RunInputHook()
 end
 
 function PySimpleObject_GetValue(::Type{T}, o) where {T}
-    Base.GC.@preserve o UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!]
+    Base.GC.@preserve o @ft UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!]
 end
 
 # FAST REFCOUNTING

src/C/pointers.jl

@@ -77,6 +77,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyType_Ready => (PyPtr,) => Cint,
     :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr,
     :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr,
+    :PyType_GetFlags => (PyPtr,) => Culong,
+    :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid},
     # MAPPING
     :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint,
     :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint,

src/JlWrap/C.jl

@@ -12,6 +12,12 @@ using Serialization: serialize, deserialize
     weaklist::C.PyPtr = C_NULL
 end
 
+@kwdef struct PyJuliaValueObjectFT
+    ob_base::C.PyObjectFT = C.PyObjectFT()
+    value::Int = 0
+    weaklist::C.PyPtr = C_NULL
+end
+
 const PyJuliaBase_Type = Ref(C.PyNULL)
 
 # we store the actual julia values here
@@ -21,21 +27,24 @@ const PYJLVALUES = []
 const PYJLFREEVALUES = Int[]
 
 function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr)
-    o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
+    alloc = C.PyType_GetSlot(t, C.Py_tp_alloc)
+    alloc == C_NULL && return C.PyNULL
+    o = ccall(alloc, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
     o == C.PyNULL && return C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).value[] = 0
+    C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
+    C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = 0
     return o
 end
 
 function _pyjl_dealloc(o::C.PyPtr)
-    idx = UnsafePtr{PyJuliaValueObject}(o).value[]
+    idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[]
     if idx != 0
         PYJLVALUES[idx] = nothing
         push!(PYJLFREEVALUES, idx)
     end
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o)
-    ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o)
+    (C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) == C.PyNULL || C.PyObject_ClearWeakRefs(o)
+    freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free)
+    freeptr == C_NULL || ccall(freeptr, Cvoid, (C.PyPtr,), o)
     nothing
 end
 
@@ -319,7 +328,7 @@ function init_c()
         C.PyMemberDef(
             name = pointer(_pyjlbase_weaklistoffset_name),
             typ = C.Py_T_PYSSIZET,
-            offset = fieldoffset(PyJuliaValueObject, 3),
+            offset = (C.@ft fieldoffset(PyJuliaValueObject, 3)),
             flags = C.Py_READONLY,
         ),
         C.PyMemberDef(), # NULL terminator
@@ -341,7 +350,7 @@ function init_c()
     # Create PyType_Spec
     _pyjlbase_spec[] = C.PyType_Spec(
         name = pointer(_pyjlbase_name),
-        basicsize = sizeof(PyJuliaValueObject),
+        basicsize = (C.@ft sizeof(PyJuliaValueObject)),
         flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG,
         slots = pointer(_pyjlbase_slots),
     )
@@ -358,13 +367,13 @@ function __init__()
     init_c()
 end
 
-PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0
+PyJuliaValue_IsNull(o) = Base.GC.@preserve o (C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]) == 0
 
-PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]]
+PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[(C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[])]
 
 PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin
     o = C.asptr(_o)
-    idx = UnsafePtr{PyJuliaValueObject}(o).value[]
+    idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[]
     if idx == 0
         if isempty(PYJLFREEVALUES)
             push!(PYJLVALUES, v)
@@ -373,7 +382,7 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin
             idx = pop!(PYJLFREEVALUES)
             PYJLVALUES[idx] = v
         end
-        UnsafePtr{PyJuliaValueObject}(o).value[] = idx
+        C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = idx
     else
         PYJLVALUES[idx] = v
     end

test/Compat.jl

@@ -50,10 +50,11 @@ end
 end
 
 @testitem "PyCall.jl" begin
-    if (get(ENV, "CI", "") != "") && (ENV["JULIA_PYTHONCALL_EXE"] == "python")
+    if (get(ENV, "CI", "") != "") && (ENV["JULIA_PYTHONCALL_EXE"] == "python") && !PythonCall.C.CTX.is_free_threaded
         # Only run this test when we can guarantee PyCall and PythonCall are using the
         # same Python. Currently this only runs in CI, and if PythonCall is using the
-        # system Python installation.
+        # system Python installation. Also PyCall is not compatible with free-threaded
+        # python so we skip this too.
         using PyCall
         # Check they are indeed using the same Python.
         @test Base.get_extension(PythonCall, :PyCallExt).SAME[]