Skip to content

Commit 8cb3d3f

Browse files
staticlibsstaticlibs
committed
Support loading native lib directly from FS
This PR is a continuation of #447 PR to allow using `duckdb_jdbc-x.x.x.x-nolib.jar` along with a JNI native library, that is loaded directly from file system. It extends the idea from #421 (and supersedes it) implementing the following logic: 1. if the driver JAR has a bundled native library (for current JVM os/arch), then this library will be unpacked to the temporary directory and loaded from there. If the library cannot be unpacked or loaded - there is no fallback to other methods (it is expected that `-nolib` JAR is used for other loading methods) 2. if the driver JAR does not hava a native library bundled inside it, then it will try to load it with: ```java System.loadLibrary("duckdb_java"); ``` This call will search the library in `java.library.path` (using a platform-specific file name like `duckdb_java.dll`) or will use other methods defined by the host application. 3. if the native library cannot be loaded by name, then the driver will check whether a file with the DuckDB internal naming (like `libduckdb_java.so_linux_amd64`) exists in file system next to the driver JAR (in the same directory). If the library file is found there - then the driver will attempt to load it as the last resort. Testing: new test added that covers loading from the same dir and loading by name. Fixes: #444
1 parent d00260e commit 8cb3d3f

File tree

6 files changed

+222
-56
lines changed

6 files changed

+222
-56
lines changed

‎CMakeLists.txt‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -560,10 +560,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
560560
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
561561
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
562562

563-
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
563+
add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
564564
MANIFEST META-INF/MANIFEST.MF
565565
GENERATE_NATIVE_HEADERS duckdb-native)
566-
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
566+
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
567567

568568

569569
# main shared lib compilation
@@ -654,7 +654,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
654654

655655
add_custom_command(
656656
OUTPUT dummy_jdbc_target
657-
DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
657+
DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
658+
COMMAND ${CMAKE_COMMAND} -E copy
659+
duckdb_jdbc_nolib.jar
660+
duckdb_jdbc.jar
658661
COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
659662
$<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
660663

‎CMakeLists.txt.in‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,10 @@ file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
8686
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
8787
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
8888

89-
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
89+
add_jar(duckdb_jdbc_nolib ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
9090
MANIFEST META-INF/MANIFEST.MF
9191
GENERATE_NATIVE_HEADERS duckdb-native)
92-
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc)
92+
add_jar(duckdb_jdbc_tests ${JAVA_TEST_FILES} INCLUDE_JARS duckdb_jdbc_nolib)
9393

9494

9595
# main shared lib compilation
@@ -180,7 +180,10 @@ set_target_properties(duckdb_java PROPERTIES PREFIX "lib")
180180

181181
add_custom_command(
182182
OUTPUT dummy_jdbc_target
183-
DEPENDS duckdb_java duckdb_jdbc duckdb_jdbc_tests
183+
DEPENDS duckdb_java duckdb_jdbc_nolib duckdb_jdbc_tests
184+
COMMAND ${CMAKE_COMMAND} -E copy
185+
duckdb_jdbc_nolib.jar
186+
duckdb_jdbc.jar
184187
COMMAND ${Java_JAR_EXECUTABLE} uf duckdb_jdbc.jar -C
185188
$<TARGET_FILE_DIR:duckdb_java> $<TARGET_FILE_NAME:duckdb_java>)
186189

‎src/main/java/org/duckdb/DuckDBNative.java‎

Lines changed: 112 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,134 @@
11
package org.duckdb;
22

3-
import java.io.File;
3+
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
4+
5+
import java.io.FileNotFoundException;
46
import java.io.IOException;
57
import java.io.InputStream;
68
import java.math.BigDecimal;
9+
import java.net.URI;
710
import java.net.URL;
811
import java.nio.ByteBuffer;
912
import java.nio.file.Files;
1013
import java.nio.file.Path;
1114
import java.nio.file.Paths;
12-
import java.nio.file.StandardCopyOption;
15+
import java.security.CodeSource;
16+
import java.security.ProtectionDomain;
1317
import java.sql.SQLException;
1418
import java.util.Properties;
1519

1620
final class DuckDBNative {
21+
22+
private static final String ARCH_X86_64 = "amd64";
23+
private static final String ARCH_AARCH64 = "arm64";
24+
private static final String ARCH_UNIVERSAL = "universal";
25+
26+
private static final String OS_WINDOWS = "windows";
27+
private static final String OS_MACOS = "osx";
28+
private static final String OS_LINUX = "linux";
29+
1730
static {
1831
try {
19-
String os_name = "";
20-
String os_arch;
21-
String os_name_detect = System.getProperty("os.name").toLowerCase().trim();
22-
String os_arch_detect = System.getProperty("os.arch").toLowerCase().trim();
23-
switch (os_arch_detect) {
24-
case "x86_64":
25-
case "amd64":
26-
os_arch = "amd64";
27-
break;
28-
case "aarch64":
29-
case "arm64":
30-
os_arch = "arm64";
31-
break;
32-
case "i386":
33-
os_arch = "i386";
34-
break;
35-
default:
36-
throw new IllegalStateException("Unsupported system architecture");
37-
}
38-
if (os_name_detect.startsWith("windows")) {
39-
os_name = "windows";
40-
} else if (os_name_detect.startsWith("mac")) {
41-
os_name = "osx";
42-
os_arch = "universal";
43-
} else if (os_name_detect.startsWith("linux")) {
44-
os_name = "linux";
45-
}
46-
String lib_res_name = "/libduckdb_java.so"
47-
+ "_" + os_name + "_" + os_arch;
48-
49-
Path lib_file = Files.createTempFile("libduckdb_java", ".so");
50-
URL lib_res = DuckDBNative.class.getResource(lib_res_name);
51-
if (lib_res == null) {
52-
System.load(Paths.get("build/debug", lib_res_name).normalize().toAbsolutePath().toString());
53-
} else {
54-
try (final InputStream lib_res_input_stream = lib_res.openStream()) {
55-
Files.copy(lib_res_input_stream, lib_file, StandardCopyOption.REPLACE_EXISTING);
56-
}
57-
new File(lib_file.toString()).deleteOnExit();
58-
System.load(lib_file.toAbsolutePath().toString());
59-
}
60-
} catch (IOException e) {
32+
loadNativeLibrary();
33+
} catch (Exception e) {
6134
throw new RuntimeException(e);
6235
}
6336
}
37+
38+
private static void loadNativeLibrary() throws Exception {
39+
String libName = nativeLibName();
40+
URL libRes = DuckDBNative.class.getResource("/" + libName);
41+
42+
// The current JAR has a native library bundled, in this case we unpack and load it.
43+
// There is no fallback if the unpacking or loading fails. We expect that only
44+
// the '-nolib' JAR can be used with an external native lib
45+
if (null != libRes) {
46+
unpackAndLoad(libRes);
47+
return;
48+
}
49+
50+
// There is no native library inside the JAR file, so we try to load it by name
51+
try {
52+
System.loadLibrary("duckdb_java");
53+
} catch (UnsatisfiedLinkError e) {
54+
// Native library cannot be loaded by name using ordinary JVM mechanisms, we try to load it directly
55+
// from FS - from the same directory where the current JAR resides
56+
try {
57+
loadFromCurrentJarDir(libName);
58+
} catch (Throwable t) {
59+
e.printStackTrace();
60+
throw new IllegalStateException(t);
61+
}
62+
}
63+
}
64+
65+
private static String cpuArch() {
66+
String prop = System.getProperty("os.arch").toLowerCase().trim();
67+
switch (prop) {
68+
case "x86_64":
69+
case "amd64":
70+
return ARCH_X86_64;
71+
case "aarch64":
72+
case "arm64":
73+
return ARCH_AARCH64;
74+
default:
75+
throw new IllegalStateException("Unsupported system architecture: '" + prop + "'");
76+
}
77+
}
78+
79+
static String osName() {
80+
String prop = System.getProperty("os.name").toLowerCase().trim();
81+
if (prop.startsWith("windows")) {
82+
return OS_WINDOWS;
83+
} else if (prop.startsWith("mac")) {
84+
return OS_MACOS;
85+
} else if (prop.startsWith("linux")) {
86+
return OS_LINUX;
87+
} else {
88+
throw new IllegalStateException("Unsupported OS: '" + prop + "'");
89+
}
90+
}
91+
92+
static String nativeLibName() {
93+
String os = osName();
94+
final String arch;
95+
if (OS_MACOS.equals(os)) {
96+
arch = ARCH_UNIVERSAL;
97+
} else {
98+
arch = cpuArch();
99+
}
100+
return "libduckdb_java.so_" + os + "_" + arch;
101+
}
102+
103+
static Path currentJarDir() throws Exception {
104+
ProtectionDomain pd = DuckDBNative.class.getProtectionDomain();
105+
CodeSource cs = pd.getCodeSource();
106+
URL loc = cs.getLocation();
107+
URI uri = loc.toURI();
108+
Path jarPath = Paths.get(uri);
109+
Path dirPath = jarPath.getParent();
110+
return dirPath.toRealPath();
111+
}
112+
113+
private static void unpackAndLoad(URL nativeLibRes) throws IOException {
114+
Path tmpFile = Files.createTempFile("libduckdb_java", ".so");
115+
try (InputStream is = nativeLibRes.openStream()) {
116+
Files.copy(is, tmpFile, REPLACE_EXISTING);
117+
}
118+
tmpFile.toFile().deleteOnExit();
119+
System.load(tmpFile.toAbsolutePath().toString());
120+
}
121+
122+
private static void loadFromCurrentJarDir(String libName) throws Exception {
123+
Path dir = currentJarDir();
124+
Path libPath = dir.resolve(libName);
125+
if (Files.exists(libPath)) {
126+
System.load(libPath.toAbsolutePath().toString());
127+
} else {
128+
throw new FileNotFoundException("DuckDB JNI library not found, path: '" + libPath.toAbsolutePath() + "'");
129+
}
130+
}
131+
64132
// We use zero-length ByteBuffer-s as a hacky but cheap way to pass C++ pointers
65133
// back and forth
66134

‎src/test/java/org/duckdb/TestDuckDBJDBC.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3155,11 +3155,12 @@ public static void main(String[] args) throws Exception {
31553155
Class<?> clazz = Class.forName("org.duckdb." + arg1);
31563156
statusCode = runTests(new String[0], clazz);
31573157
} else {
3158-
statusCode = runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
3159-
TestAppenderCollection2D.class, TestAppenderComposite.class,
3160-
TestSingleValueAppender.class, TestBatch.class, TestBindings.class, TestClosure.class,
3161-
TestExtensionTypes.class, TestSpatial.class, TestParameterMetadata.class,
3162-
TestPrepare.class, TestResults.class, TestSessionInit.class, TestTimestamp.class);
3158+
statusCode =
3159+
runTests(args, TestDuckDBJDBC.class, TestAppender.class, TestAppenderCollection.class,
3160+
TestAppenderCollection2D.class, TestAppenderComposite.class, TestSingleValueAppender.class,
3161+
TestBatch.class, TestBindings.class, TestClosure.class, TestExtensionTypes.class,
3162+
TestNoLib.class, TestSpatial.class, TestParameterMetadata.class, TestPrepare.class,
3163+
TestResults.class, TestSessionInit.class, TestTimestamp.class);
31633164
}
31643165
System.exit(statusCode);
31653166
}

‎src/test/java/org/duckdb/TestNoLib.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package org.duckdb;
2+
3+
import static java.util.Arrays.asList;
4+
5+
import java.io.File;
6+
import java.nio.file.Files;
7+
import java.nio.file.Path;
8+
import java.nio.file.Paths;
9+
import java.sql.SQLException;
10+
import org.duckdb.test.TempDirectory;
11+
12+
public class TestNoLib {
13+
14+
private static Path javaExe() {
15+
String javaHomeProp = System.getProperty("java.home");
16+
Path javaHome = Paths.get(javaHomeProp);
17+
boolean isWindows = "windows".equals(DuckDBNative.osName());
18+
return isWindows ? javaHome.resolve("bin/java.exe") : javaHome.resolve("bin/java");
19+
}
20+
21+
private static void runQuickTest(Path currentJarDir) throws Exception {
22+
String dir = currentJarDir.toAbsolutePath().toString();
23+
ProcessBuilder pb = new ProcessBuilder(javaExe().toAbsolutePath().toString(),
24+
"-Djava.library.path=" + currentJarDir.toAbsolutePath(), "-cp",
25+
dir + File.separator + "duckdb_jdbc_tests.jar" + File.pathSeparator +
26+
dir + File.separator + "duckdb_jdbc_nolib.jar",
27+
"org.duckdb.TestDuckDBJDBC", "test_spatial_POINT_2D")
28+
.inheritIO();
29+
int code = pb.start().waitFor();
30+
if (0 != code) {
31+
throw new RuntimeException("Spawned test failure, code: " + code);
32+
}
33+
}
34+
35+
private static String platformLibName() throws Exception {
36+
String os = DuckDBNative.osName();
37+
switch (os) {
38+
case "windows":
39+
return "duckdb_java.dll";
40+
case "osx":
41+
return "libduckdb_java.dylib";
42+
case "linux":
43+
return "libduckdb_java.so";
44+
default:
45+
throw new SQLException("Unsupported OS: " + os);
46+
}
47+
}
48+
private static Path nativeLibPathInBuildTree(Path buildDir) throws SQLException {
49+
String libName = DuckDBNative.nativeLibName();
50+
Path libPath = buildDir.resolve(libName);
51+
if (Files.exists(libPath)) {
52+
return libPath;
53+
}
54+
for (String subdirName : asList("Release", "Debug", "RelWithDebInfo")) {
55+
Path dir = buildDir.resolve(subdirName);
56+
Path libPathSubdir = dir.resolve(libName);
57+
if (Files.exists(libPathSubdir)) {
58+
return libPathSubdir;
59+
}
60+
}
61+
throw new SQLException("Native lib not found in build tree, name: '" + libName + "'");
62+
}
63+
64+
public static void test_nolib_next_to_jar() throws Exception {
65+
try (TempDirectory td = new TempDirectory()) {
66+
Path dir = DuckDBNative.currentJarDir();
67+
Path nativeLib = nativeLibPathInBuildTree(dir);
68+
Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar"));
69+
Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar"));
70+
Files.copy(nativeLib, td.path().resolve(nativeLib.getFileName()));
71+
System.out.println();
72+
System.out.println("----");
73+
runQuickTest(td.path());
74+
System.out.println("----");
75+
}
76+
}
77+
78+
public static void test_nolib_by_name() throws Exception {
79+
try (TempDirectory td = new TempDirectory()) {
80+
Path dir = DuckDBNative.currentJarDir();
81+
Path nativeLib = nativeLibPathInBuildTree(dir);
82+
Files.copy(dir.resolve("duckdb_jdbc_nolib.jar"), td.path().resolve("duckdb_jdbc_nolib.jar"));
83+
Files.copy(dir.resolve("duckdb_jdbc_tests.jar"), td.path().resolve("duckdb_jdbc_tests.jar"));
84+
Files.copy(nativeLib, td.path().resolve(platformLibName()));
85+
System.out.println();
86+
System.out.println("----");
87+
runQuickTest(td.path());
88+
System.out.println("----");
89+
}
90+
}
91+
}

‎src/test/java/org/duckdb/test/Runner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static int runTests(String[] args, Class<?>... testClasses) {
4040
boolean anyFailed = false;
4141
for (Method m : methods) {
4242
if (m.getName().startsWith("test_")) {
43-
if (quick_run && m.getName().startsWith("test_lots_")) {
43+
if (quick_run && (m.getName().startsWith("test_lots_") || m.getName().startsWith("test_nolib_"))) {
4444
continue;
4545
}
4646
if (specific_test != null && !m.getName().contains(specific_test)) {

0 commit comments

Comments
 (0)