Skip to content

Lookup ccall symbols in internal libraries first #49010

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 1, 2023
Merged

Lookup ccall symbols in internal libraries first #49010

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
95f4376
Lookup `ccall` symbols in local internal libraries first
topolarity Mar 14, 2023
bbd3639
dlload: use RTLD_LOCAL when looking up internal libraries
topolarity Apr 17, 2023
d22ca94
Merge branch 'master' into dlvsym-ccall
staticfloat May 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 13 additions & 17 deletions src/ccall.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ static bool runtime_sym_gvs(jl_codectx_t &ctx, const char *f_lib, const char *f_
bool runtime_lib = false;
GlobalVariable *libptrgv;
jl_codegen_params_t::SymMapGV *symMap;
#ifdef _OS_WINDOWS_
if ((intptr_t)f_lib == (intptr_t)JL_EXE_LIBNAME) {
libptrgv = prepare_global_in(M, jlexe_var);
symMap = &ctx.emission_context.symMapExe;
Expand All @@ -74,9 +73,7 @@ static bool runtime_sym_gvs(jl_codectx_t &ctx, const char *f_lib, const char *f_
libptrgv = prepare_global_in(M, jldll_var);
symMap = &ctx.emission_context.symMapDll;
}
else
#endif
if (f_lib == NULL) {
else if (f_lib == NULL) {
libptrgv = jl_emit_RTLD_DEFAULT_var(M);
symMap = &ctx.emission_context.symMapDefault;
}
Expand Down Expand Up @@ -631,16 +628,12 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va
std::string iname("i");
iname += f_name;
if (jl_dlsym(jl_libjulia_internal_handle, iname.c_str(), &symaddr, 0)) {
#ifdef _OS_WINDOWS_
f_lib = JL_LIBJULIA_INTERNAL_DL_LIBNAME;
#endif
f_name = jl_symbol_name(jl_symbol(iname.c_str()));
}
#ifdef _OS_WINDOWS_
else {
f_lib = jl_dlfind_win32(f_name);
f_lib = jl_dlfind(f_name);
}
#endif
}
}
else if (jl_is_cpointer_type(jl_typeof(ptr))) {
Expand Down Expand Up @@ -726,7 +719,8 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg
void *symaddr;

void* libsym = jl_get_library_(sym.f_lib, 0);
if (!libsym || !jl_dlsym(libsym, sym.f_name, &symaddr, 0)) {
int symbol_found = jl_dlsym(libsym, sym.f_name, &symaddr, 0);
if (!libsym || !symbol_found) {
// Error mode, either the library or the symbol couldn't be find during compiletime.
// Fallback to a runtime symbol lookup.
res = runtime_sym_lookup(ctx, cast<PointerType>(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f);
Expand Down Expand Up @@ -1381,18 +1375,19 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
if ((uintptr_t)fptr == ptr)
return true;
if (f_lib) {
#ifdef _OS_WINDOWS_
if ((f_lib == JL_EXE_LIBNAME) || // preventing invalid pointer access
(f_lib == JL_LIBJULIA_INTERNAL_DL_LIBNAME) ||
(f_lib == JL_LIBJULIA_DL_LIBNAME) ||
(!strcmp(f_lib, jl_crtdll_basename))) {
(f_lib == JL_LIBJULIA_DL_LIBNAME)) {
// libjulia-like
}
else
#ifdef _OS_WINDOWS_
if (strcmp(f_lib, jl_crtdll_basename) == 0) {
// libjulia-like
}
else
return false;
#else
return false;
#endif
return false;
}
return f_name && f_name == name;
};
Expand Down Expand Up @@ -2082,7 +2077,8 @@ jl_cgval_t function_sig_t::emit_a_ccall(
else {
void *symaddr;
void *libsym = jl_get_library_(symarg.f_lib, 0);
if (!libsym || !jl_dlsym(libsym, symarg.f_name, &symaddr, 0)) {
int symbol_found = jl_dlsym(libsym, symarg.f_name, &symaddr, 0);
if (!libsym || !symbol_found) {
++DeferredCCallLookups;
// either the library or the symbol could not be found, place a runtime
// lookup here instead.
Expand Down
2 changes: 0 additions & 2 deletions src/codegen.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8686,11 +8686,9 @@ static void init_jit_functions(void)
{
add_named_global(jlstack_chk_guard_var, &__stack_chk_guard);
add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle);
#ifdef _OS_WINDOWS_
add_named_global(jlexe_var, &jl_exe_handle);
add_named_global(jldll_var, &jl_libjulia_handle);
add_named_global(jldlli_var, &jl_libjulia_internal_handle);
#endif
auto size2pjlvalue = [](Type *T_size) -> Type * {
return get_pjlvalue(T_size->getContext());
};
Expand Down
48 changes: 24 additions & 24 deletions src/dlload.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ JL_DLLEXPORT JL_NO_SANITIZE void *jl_dlopen(const char *filename, unsigned flags
if (!dlopen)
return NULL;
void *libdl_handle = dlopen("libdl.so", RTLD_NOW | RTLD_NOLOAD);
assert(libdl_handle);
dlopen = (dlopen_prototype*)dlsym(libdl_handle, "dlopen");
dlclose(libdl_handle);
assert(dlopen);
Expand Down Expand Up @@ -239,6 +240,25 @@ JL_DLLEXPORT int jl_dlclose(void *handle) JL_NOTSAFEPOINT
#endif
}

void *jl_find_dynamic_library_by_addr(void *symbol) {
void *handle;
#ifdef _OS_WINDOWS_
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCWSTR)symbol,
(HMODULE*)&handle)) {
jl_error("could not load base module");
}
#else
Dl_info info;
if (!dladdr(symbol, &info) || !info.dli_fname) {
jl_error("could not load base module");
}
handle = dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD);
dlclose(handle); // Undo ref count increment from `dlopen`
#endif
return handle;
}

JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, int throw_err)
{
char path[PATHBUF], relocated[PATHBUF];
Expand All @@ -255,26 +275,6 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags,
int n_extensions = endswith_extension(modname) ? 1 : N_EXTENSIONS;
int ret;

/*
this branch returns handle of libjulia-internal
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it was actually trying to trigger this case however, which is not as much of a trick and not quite what it claimed to be doing. I assume we could put this back?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like it, yes. Putting it back makes CBinding work again, as expected. @topolarity what was the reason to remove this functionality?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I made a mistake and assumed that downstream code would not end up here after all of the runtime-internal uses were re-factored into jl_find_library_by_addr.

PR open to fix: #49611

*/
if (modname == NULL) {
#ifdef _OS_WINDOWS_
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCWSTR)(uintptr_t)(&jl_load_dynamic_library),
(HMODULE*)&handle)) {
jl_error("could not load base module");
}
#else
Dl_info info;
if (!dladdr((void*)(uintptr_t)&jl_load_dynamic_library, &info) || !info.dli_fname) {
jl_error("could not load base module");
}
handle = dlopen(info.dli_fname, RTLD_NOW);
#endif
goto done;
}

abspath = jl_isabspath(modname);
is_atpath = 0;

Expand Down Expand Up @@ -421,9 +421,8 @@ JL_DLLEXPORT int jl_dlsym(void *handle, const char *symbol, void ** value, int t
return symbol_found;
}

#ifdef _OS_WINDOWS_
//Look for symbols in win32 libraries
JL_DLLEXPORT const char *jl_dlfind_win32(const char *f_name)
// Look for symbols in internal libraries
JL_DLLEXPORT const char *jl_dlfind(const char *f_name)
{
void * dummy;
if (jl_dlsym(jl_exe_handle, f_name, &dummy, 0))
Expand All @@ -432,6 +431,7 @@ JL_DLLEXPORT const char *jl_dlfind_win32(const char *f_name)
return JL_LIBJULIA_INTERNAL_DL_LIBNAME;
if (jl_dlsym(jl_libjulia_handle, f_name, &dummy, 0))
return JL_LIBJULIA_DL_LIBNAME;
#ifdef _OS_WINDOWS_
if (jl_dlsym(jl_kernel32_handle, f_name, &dummy, 0))
return "kernel32";
if (jl_dlsym(jl_crtdll_handle, f_name, &dummy, 0)) // Prefer crtdll over ntdll
Expand All @@ -440,6 +440,7 @@ JL_DLLEXPORT const char *jl_dlfind_win32(const char *f_name)
return "ntdll";
if (jl_dlsym(jl_winsock_handle, f_name, &dummy, 0))
return "ws2_32";
#endif
// additional common libraries (libc?) could be added here, but in general,
// it is better to specify the library explicitly in the code. This exists
// mainly to ease compatibility with linux, and for libraries that don't
Expand All @@ -451,7 +452,6 @@ JL_DLLEXPORT const char *jl_dlfind_win32(const char *f_name)
// which defaults to jl_libjulia_internal_handle, where we won't find it, and
// will throw the appropriate error.
}
#endif

#ifdef __cplusplus
}
Expand Down
8 changes: 2 additions & 6 deletions src/init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -763,15 +763,11 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel)
void *stack_lo, *stack_hi;
jl_init_stack_limits(1, &stack_lo, &stack_hi);

jl_libjulia_internal_handle = jl_load_dynamic_library(NULL, JL_RTLD_DEFAULT, 1);
jl_libjulia_internal_handle = jl_find_dynamic_library_by_addr(&jl_load_dynamic_library);
jl_libjulia_handle = jl_find_dynamic_library_by_addr(&jl_any_type);
#ifdef _OS_WINDOWS_
jl_exe_handle = GetModuleHandleA(NULL);
jl_RTLD_DEFAULT_handle = jl_libjulia_internal_handle;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
(LPCWSTR)&jl_any_type,
(HMODULE*)&jl_libjulia_handle)) {
jl_error("could not load base module");
}
jl_ntdll_handle = jl_dlopen("ntdll.dll", JL_RTLD_NOLOAD); // bypass julia's pathchecking for system dlls
jl_kernel32_handle = jl_dlopen("kernel32.dll", JL_RTLD_NOLOAD);
jl_crtdll_handle = jl_dlopen(jl_crtdll_name, JL_RTLD_NOLOAD);
Expand Down
9 changes: 5 additions & 4 deletions src/julia_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1254,11 +1254,11 @@ JL_DLLEXPORT uint64_t jl_rand(void) JL_NOTSAFEPOINT;
JL_DLLEXPORT void jl_srand(uint64_t) JL_NOTSAFEPOINT;
JL_DLLEXPORT void jl_init_rand(void);

JL_DLLEXPORT extern void *jl_exe_handle;
JL_DLLEXPORT extern void *jl_libjulia_handle;
JL_DLLEXPORT extern void *jl_libjulia_internal_handle;
JL_DLLEXPORT extern void *jl_RTLD_DEFAULT_handle;
#if defined(_OS_WINDOWS_)
JL_DLLEXPORT extern void *jl_exe_handle;
JL_DLLEXPORT extern void *jl_libjulia_handle;
JL_DLLEXPORT extern const char *jl_crtdll_basename;
extern void *jl_ntdll_handle;
extern void *jl_kernel32_handle;
Expand All @@ -1268,6 +1268,7 @@ void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOINT;
#endif

JL_DLLEXPORT void *jl_get_library_(const char *f_lib, int throw_err);
void *jl_find_dynamic_library_by_addr(void *symbol);
#define jl_get_library(f_lib) jl_get_library_(f_lib, 1)
JL_DLLEXPORT void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *hnd);
JL_DLLEXPORT void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name);
Expand All @@ -1277,11 +1278,11 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline(
jl_unionall_t *env, jl_value_t **vals);


// Windows only
// Special filenames used to refer to internal julia libraries
#define JL_EXE_LIBNAME ((const char*)1)
#define JL_LIBJULIA_DL_LIBNAME ((const char*)2)
#define JL_LIBJULIA_INTERNAL_DL_LIBNAME ((const char*)3)
JL_DLLEXPORT const char *jl_dlfind_win32(const char *name);
JL_DLLEXPORT const char *jl_dlfind(const char *name);

// libuv wrappers:
JL_DLLEXPORT int jl_fs_rename(const char *src_path, const char *dst_path);
Expand Down
2 changes: 0 additions & 2 deletions src/runtime_ccall.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,12 @@ void *jl_get_library_(const char *f_lib, int throw_err)
{
if (f_lib == NULL)
return jl_RTLD_DEFAULT_handle;
#ifdef _OS_WINDOWS_
if (f_lib == JL_EXE_LIBNAME)
return jl_exe_handle;
if (f_lib == JL_LIBJULIA_INTERNAL_DL_LIBNAME)
return jl_libjulia_internal_handle;
if (f_lib == JL_LIBJULIA_DL_LIBNAME)
return jl_libjulia_handle;
#endif
JL_LOCK(&libmap_lock);
// This is the only operation we do on the map, which doesn't invalidate
// any references or iterators.
Expand Down
4 changes: 1 addition & 3 deletions src/runtime_intrinsics.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,10 +508,8 @@ JL_DLLEXPORT jl_value_t *jl_cglobal(jl_value_t *v, jl_value_t *ty)
else
JL_TYPECHK(cglobal, symbol, v)

#ifdef _OS_WINDOWS_
if (!f_lib)
f_lib = (char*)jl_dlfind_win32(f_name);
#endif
f_lib = (char*)jl_dlfind(f_name);

void *ptr;
jl_dlsym(jl_get_library(f_lib), f_name, &ptr, 1);
Expand Down