diff --git a/src/driver.jl b/src/driver.jl index 950ea272..6bc9717e 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -197,7 +197,7 @@ const __llvm_initialized = Ref(false) end @tracepoint "IR generation" begin - ir, compiled = irgen(job) + ir, compiled, gv_to_value = irgen(job) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -256,6 +256,7 @@ const __llvm_initialized = Ref(false) dyn_ir, dyn_meta = codegen(:llvm, CompilerJob(dyn_job; config)) dyn_entry_fn = LLVM.name(dyn_meta.entry) merge!(compiled, dyn_meta.compiled) + merge!(gv_to_value, dyn_meta.gv_to_value) @assert context(dyn_ir) == context(ir) link!(ir, dyn_ir) changed = true @@ -319,6 +320,20 @@ const __llvm_initialized = Ref(false) end end + # TODO: Move this to somewhere else? + @tracepoint "Resolve relocations eagerly" for gv in globals(ir) + name = LLVM.name(gv) + init = get(gv_to_value, name, nothing) + if init !== nothing + if initializer(gv) !== nothing + # TODO: How is this happening we should have stripped initializers earlier + @show string(initializer(gv)), init + end + val = const_inttoptr(ConstantInt(reinterpret(UInt, init)), LLVM.PointerType()) + initializer!(gv, val) + end + end + @tracepoint "IR post-processing" begin # mark the kernel entry-point functions (optimization may need it) if job.config.kernel @@ -422,7 +437,7 @@ const __llvm_initialized = Ref(false) @tracepoint "verification" verify(ir) end - return ir, (; entry, compiled) + return ir, (; entry, compiled, gv_to_value) end @locked function emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, diff --git a/src/execution.jl b/src/execution.jl index 9b4940a7..78d05736 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -253,6 +253,13 @@ end if !ondisk_hit && path !== nothing && disk_cache_enabled() @debug "Writing out on-disk cache" job path mkpath(dirname(path)) + if haskey(asm[2], :gv_to_value) + # TODO: Serializer cannot handle Core.IntrinsicFunction + # We kinda want Julia to serialize the values in `gv_to_value` in the pkgimg and us just having to store an index + # for now we just empty them out + # We would need to remove the initializers from LLVM IR as well to be correct, and then link these in at runtime + empty!(asm[2].gv_to_value) + end entry = DiskCacheEntry(src.specTypes, cfg, asm) # atomic write to disk diff --git a/src/irgen.jl b/src/irgen.jl index a7c36a60..5149e9f0 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -1,7 +1,7 @@ # LLVM IR generation function irgen(@nospecialize(job::CompilerJob)) - mod, compiled = @tracepoint "emission" compile_method_instance(job) + mod, compiled, gv_to_value = @tracepoint "emission" compile_method_instance(job) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -55,6 +55,11 @@ function irgen(@nospecialize(job::CompilerJob)) new_name = safe_name(old_name) if old_name != new_name LLVM.name!(val, new_name) + val = get(gv_to_value, old_name, nothing) + if val !== nothing + delete!(gv_to_value, old_name) + gv_to_value[new_name] = val + end end end @@ -120,7 +125,7 @@ function irgen(@nospecialize(job::CompilerJob)) can_throw(job) || lower_throw!(mod) end - return mod, compiled + return mod, compiled, gv_to_value end diff --git a/src/jlgen.jl b/src/jlgen.jl index 330cf7f8..87786ee4 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -672,6 +672,28 @@ end CompilationPolicyExtern = 1 end +const AL_N_INLINE = 29 +# mirrows arraylist_t +mutable struct ArrayList + len::Csize_t + max::Csize_t + items::Ptr{Ptr{Cvoid}} + _space::NTuple{AL_N_INLINE, Ptr{Cvoid}} + + function ArrayList() + list = new(0, AL_N_INLINE, Ptr{Ptr{Cvoid}}(C_NULL), ntuple(_->Ptr{Cvoid}(C_NULL), AL_N_INLINE)) + list.items = Base.pointer_from_objref(list) + fieldoffset(typeof(list), 4) + + finalizer(list) do list + if list.items != Base.pointer_from_objref(list) + fieldoffset(typeof(list), 4) + Libc.free(list.items) + end + end + return list + end +end + + """ precompile(job::CompilerJob) @@ -786,25 +808,73 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) cache_gbl = nothing end + gv_to_value = Dict{String, Ptr{Cvoid}}() + + # The caller is responsible for initializing global variables that + # point to global values or bindings with their address in memory. + # For Julia < v"1.13" to enable relocation we strip out the initializers here. if VERSION >= v"1.13.0-DEV.623" - # Since Julia 1.13, the caller is responsible for initializing global variables that - # point to global values or bindings with their address in memory. num_gvars = Ref{Csize_t}(0) @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, - C_NULL::Ptr{Cvoid})::Nothing + C_NULL::Ptr{Cvoid})::Nothing gvs = Vector{Ptr{LLVM.API.LLVMOpaqueValue}}(undef, num_gvars[]) @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, - gvs::Ptr{LLVM.API.LLVMOpaqueValue})::Nothing + gvs::Ptr{LLVM.API.LLVMOpaqueValue})::Nothing + inits = Vector{Ptr{Cvoid}}(undef, num_gvars[]) @ccall jl_get_llvm_gv_inits(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, inits::Ptr{Cvoid})::Nothing for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) - val = const_inttoptr(ConstantInt(Int64(init)), LLVM.PointerType()) - initializer!(gv, val) + gv_to_value[LLVM.name(gv)] = init + end + else + # Prior to this version of Julia we only had access to the values that the global variables + # were initialized with, so we have to match them up manually. + # get the global values + if VERSION >= v"1.12.0-DEV.1703" + num_gvars = Ref{Csize_t}(0) + @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, + C_NULL::Ptr{Cvoid})::Nothing + gvalues = Vector{Ptr{Cvoid}}(undef, num_gvars[]) + @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, num_gvars::Ptr{Csize_t}, + gvalues::Ptr{Cvoid})::Nothing + else + # Sigh on older version of Julia we have to use `arraylist_t` which doesn't have a Julia API. + gvars = ArrayList() + GC.@preserve gvars begin + p_gvars = Base.pointer_from_objref(gvars) + @ccall jl_get_llvm_gvs(native_code::Ptr{Cvoid}, p_gvars::Ptr{Cvoid})::Nothing + gvalues = Vector{Ptr{Cvoid}}(undef, gvars.len) + for i in 1:gvars.len + gvalues[i] = unsafe_load(gvars.items, i) + end + end + end + gvalues = Set(gvalues) + for gv in globals(llvm_mod) + init = LLVM.initializer(gv) + if init === nothing + continue + end + if init isa LLVM.ConstantExpr && opcode(init) == LLVM.API.LLVMIntToPtr + init = operands(init)[1] + end + if !(init isa LLVM.ConstantInt) + continue + end + ptr = reinterpret(Ptr{Cvoid}, convert(UInt, init)) + if ptr in gvalues + gv_to_value[LLVM.name(gv)] = ptr + end + LLVM.initializer!(gv, nothing) end + @assert length(gv_to_value) == length(gvalues) end + # It's valid to call Base.unsafe_pointer_to_objref on values(gv_to_value), + # but we may not be able to "easily" obtain the pointer back later. + # (Types, etc, disallow Base.pointer_from_objref on them.) if VERSION >= v"1.13.0-DEV.1120" # on sufficiently recent versions of Julia, we can query the CIs compiled. @@ -874,7 +944,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # ensure that the requested method instance was compiled @assert haskey(compiled, job.source) - return llvm_mod, compiled + return llvm_mod, compiled, gv_to_value end # partially revert JuliaLangjulia#49391