diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets
index 4f3586667..6466a1204 100644
--- a/LLama/LLamaSharp.Runtime.targets
+++ b/LLama/LLamaSharp.Runtime.targets
@@ -52,6 +52,14 @@
PreserveNewest
runtimes/win-x64/native/cuda12/ggml.dll
+
+ PreserveNewest
+ runtimes/win-x64/native/vulkan/llama.dll
+
+
+ PreserveNewest
+ runtimes/win-x64/native/vulkan/ggml.dll
+
PreserveNewest
@@ -101,6 +109,14 @@
PreserveNewest
runtimes/linux-x64/native/cuda12/libggml.so
+
+ PreserveNewest
+ runtimes/linux-x64/native/vulkan/libllama.so
+
+
+ PreserveNewest
+ runtimes/linux-x64/native/vulkan/libggml.so
+
PreserveNewest
@@ -169,6 +185,10 @@
PreserveNewest
runtimes/win-x64/native/cuda12/llava_shared.dll
+
+ PreserveNewest
+ runtimes/win-x64/native/vulkan/llava_shared.dll
+
PreserveNewest
@@ -194,5 +214,9 @@
PreserveNewest
runtimes/linux-x64/native/cuda12/libllava_shared.so
+
+ PreserveNewest
+ runtimes/linux-x64/native/vulkan/libllava_shared.so
+
\ No newline at end of file
diff --git a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs
index 5cb3b0c5a..690fa706f 100644
--- a/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs
+++ b/LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs
@@ -28,7 +28,12 @@ public IEnumerable Apply(NativeLibraryConfig.Description descrip
yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck);
}
- if(!description.UseCuda || description.AllowFallback)
+ if (description.UseVulkan)
+ {
+ yield return new NativeLibraryWithVulkan(systemInfo.VulkanVersion, description.Library, description.SkipCheck);
+ }
+
+ if((!description.UseCuda || !description.UseVulkan) || description.AllowFallback)
{
if (description.AllowFallback)
{
diff --git a/LLama/Native/Load/NativeLibraryConfig.cs b/LLama/Native/Load/NativeLibraryConfig.cs
index 478102177..02e47b695 100644
--- a/LLama/Native/Load/NativeLibraryConfig.cs
+++ b/LLama/Native/Load/NativeLibraryConfig.cs
@@ -16,6 +16,7 @@ public sealed partial class NativeLibraryConfig
private string? _libraryPath;
private bool _useCuda = true;
+ private bool _useVulkan = true;
private AvxLevel _avxLevel;
private bool _allowFallback = true;
private bool _skipCheck = false;
@@ -55,6 +56,20 @@ public NativeLibraryConfig WithCuda(bool enable = true)
_useCuda = enable;
return this;
}
+
+ ///
+ /// Configure whether to use vulkan backend if possible. Default is true.
+ ///
+ ///
+ ///
+ /// Thrown if `LibraryHasLoaded` is true.
+ public NativeLibraryConfig WithVulkan(bool enable = true)
+ {
+ ThrowIfLoaded();
+
+ _useVulkan = enable;
+ return this;
+ }
///
/// Configure the prefferred avx support level of the backend.
@@ -159,6 +174,7 @@ internal Description CheckAndGatherDescription()
path,
NativeLibraryName,
_useCuda,
+ _useVulkan,
_avxLevel,
_allowFallback,
_skipCheck,
@@ -229,7 +245,7 @@ private static bool CheckAVX512()
///
///
///
- public record Description(string? Path, NativeLibraryName Library, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck,
+ public record Description(string? Path, NativeLibraryName Library, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck,
string[] SearchDirectories)
{
///
@@ -250,6 +266,7 @@ public override string ToString()
$"- LibraryName: {Library}\n" +
$"- Path: '{Path}'\n" +
$"- PreferCuda: {UseCuda}\n" +
+ $"- PreferVulkan: {UseVulkan}\n" +
$"- PreferredAvxLevel: {avxLevelString}\n" +
$"- AllowFallback: {AllowFallback}\n" +
$"- SkipCheck: {SkipCheck}\n" +
@@ -425,6 +442,21 @@ public NativeLibraryConfigContainer WithCuda(bool enable = true)
}
return this;
}
+
+ ///
+ /// Configure whether to use vulkan backend if possible.
+ ///
+ ///
+ ///
+ /// Thrown if `LibraryHasLoaded` is true.
+ public NativeLibraryConfigContainer WithVulkan(bool enable = true)
+ {
+ foreach(var config in _configs)
+ {
+ config.WithVulkan(enable);
+ }
+ return this;
+ }
///
/// Configure the prefferred avx support level of the backend.
diff --git a/LLama/Native/Load/NativeLibraryMetadata.cs b/LLama/Native/Load/NativeLibraryMetadata.cs
index 66c546e6b..d458610b6 100644
--- a/LLama/Native/Load/NativeLibraryMetadata.cs
+++ b/LLama/Native/Load/NativeLibraryMetadata.cs
@@ -6,12 +6,13 @@ namespace LLama.Native
///
/// Which kind of library it is.
/// Whether it's compiled with cublas.
+ /// Whether it's compiled with vulkan.
/// Which AvxLevel it's compiled with.
- public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, AvxLevel AvxLevel)
+ public record class NativeLibraryMetadata(NativeLibraryName NativeLibraryName, bool UseCuda, bool UseVulkan, AvxLevel AvxLevel)
{
public override string ToString()
{
- return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, AvxLevel: {AvxLevel})";
+ return $"(NativeLibraryName: {NativeLibraryName}, UseCuda: {UseCuda}, UseVulkan: {UseVulkan}, AvxLevel: {AvxLevel})";
}
}
diff --git a/LLama/Native/Load/NativeLibraryWithAvx.cs b/LLama/Native/Load/NativeLibraryWithAvx.cs
index 7b5421b4d..52e62845f 100644
--- a/LLama/Native/Load/NativeLibraryWithAvx.cs
+++ b/LLama/Native/Load/NativeLibraryWithAvx.cs
@@ -19,7 +19,7 @@ public NativeLibraryMetadata? Metadata
{
get
{
- return new NativeLibraryMetadata(_libraryName, false, _avxLevel);
+ return new NativeLibraryMetadata(_libraryName, false, false,_avxLevel);
}
}
diff --git a/LLama/Native/Load/NativeLibraryWithCuda.cs b/LLama/Native/Load/NativeLibraryWithCuda.cs
index d3b06b864..982236771 100644
--- a/LLama/Native/Load/NativeLibraryWithCuda.cs
+++ b/LLama/Native/Load/NativeLibraryWithCuda.cs
@@ -20,7 +20,7 @@ public NativeLibraryMetadata? Metadata
{
get
{
- return new NativeLibraryMetadata(_libraryName, true, _avxLevel);
+ return new NativeLibraryMetadata(_libraryName, true, false, _avxLevel);
}
}
diff --git a/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs
index 4bde4dae1..55734fcc4 100644
--- a/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs
+++ b/LLama/Native/Load/NativeLibraryWithMacOrFallback.cs
@@ -18,7 +18,7 @@ public NativeLibraryMetadata? Metadata
{
get
{
- return new NativeLibraryMetadata(_libraryName, false, AvxLevel.None);
+ return new NativeLibraryMetadata(_libraryName, false, false, AvxLevel.None);
}
}
diff --git a/LLama/Native/Load/NativeLibraryWithVulkan.cs b/LLama/Native/Load/NativeLibraryWithVulkan.cs
new file mode 100644
index 000000000..867a52f52
--- /dev/null
+++ b/LLama/Native/Load/NativeLibraryWithVulkan.cs
@@ -0,0 +1,65 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+ ///
+ /// A native library compiled with vulkan.
+ ///
+ public class NativeLibraryWithVulkan : INativeLibrary
+ {
+ private string? _vulkanVersion;
+ private NativeLibraryName _libraryName;
+ private AvxLevel _avxLevel;
+ private bool _skipCheck;
+
+ ///
+ public NativeLibraryMetadata? Metadata
+ {
+ get
+ {
+ return new NativeLibraryMetadata(_libraryName, false, true, _avxLevel);
+ }
+ }
+
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ public NativeLibraryWithVulkan(string? vulkanVersion, NativeLibraryName libraryName, bool skipCheck)
+ {
+ _vulkanVersion = vulkanVersion;
+ _libraryName = libraryName;
+ _skipCheck = skipCheck;
+ }
+
+ ///
+ public IEnumerable Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+ {
+ // TODO: Avx level is ignored now, needs to be implemented in the future.
+ if (systemInfo.OSPlatform == OSPlatform.Windows || systemInfo.OSPlatform == OSPlatform.Linux || _skipCheck)
+ {
+ if(systemInfo.VulkanVersion != null)
+ {
+ var vulkanLibraryPath = GetVulkanPath(systemInfo, logCallback);
+ if (vulkanLibraryPath is not null)
+ {
+ yield return vulkanLibraryPath;
+ }
+ }
+ }
+ }
+
+ private string? GetVulkanPath(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+ {
+ NativeLibraryUtils.GetPlatformPathParts(systemInfo.OSPlatform, out var os, out var fileExtension, out var libPrefix);
+ var relativePath = $"runtimes/{os}/native/vulkan/{libPrefix}{_libraryName.GetLibraryName()}{fileExtension}";
+ return relativePath;
+ }
+ }
+#endif
+}
diff --git a/LLama/Native/Load/SystemInfo.cs b/LLama/Native/Load/SystemInfo.cs
index 0ffc67e91..a1cff5970 100644
--- a/LLama/Native/Load/SystemInfo.cs
+++ b/LLama/Native/Load/SystemInfo.cs
@@ -1,7 +1,9 @@
using System;
+using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text.Json;
+using System.Text.RegularExpressions;
namespace LLama.Native
{
@@ -10,7 +12,8 @@ namespace LLama.Native
///
///
///
- public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion)
+ ///
+ public record class SystemInfo(OSPlatform OSPlatform, int CudaMajorVersion, string? VulkanVersion)
{
///
/// Get the system information of the current machine.
@@ -37,8 +40,108 @@ public static SystemInfo Get()
throw new PlatformNotSupportedException();
}
- return new SystemInfo(platform, GetCudaMajorVersion());
+ return new SystemInfo(platform, GetCudaMajorVersion(), GetVulkanVersion());
}
+
+ #region Vulkan version
+ private static string? GetVulkanVersion()
+ {
+ // Get Vulkan Summary
+ string? vulkanSummary = GetVulkanSummary();
+
+ // If we have a Vulkan summary
+ if (vulkanSummary != null)
+ {
+ // Extract Vulkan version from summary
+ string? vulkanVersion = ExtractVulkanVersionFromSummary(vulkanSummary);
+
+ // If we have a Vulkan version
+ if (vulkanVersion != null)
+ {
+ // Return the Vulkan version
+ return vulkanVersion;
+ }
+ }
+
+ // Return null if we failed to get the Vulkan version
+ return null;
+ }
+
+ private static string? GetVulkanSummary()
+ {
+ // Note: on Linux, this requires `vulkan-tools` to be installed. (`sudo apt install vulkan-tools`)
+ try
+ {
+ // Set up the process start info
+ ProcessStartInfo start = new()
+ {
+ FileName = "vulkaninfo",
+ Arguments = "--summary",
+ RedirectStandardOutput = true,
+ UseShellExecute = false,
+ CreateNoWindow = true
+ };
+
+ // Start the process
+ Process process = new()
+ {
+ StartInfo = start
+ };
+ process.Start();
+
+ // Read the output to a string
+ string output = process.StandardOutput.ReadToEnd();
+
+ // Wait for the process to exit
+ process.WaitForExit();
+
+ // Return the output
+ return output;
+ }
+ catch (Exception e)
+ {
+ //Console.WriteLine(e);
+
+ // Return null if we failed to get the Vulkan version
+ return null;
+ }
+ }
+
+ static string? ExtractVulkanVersionFromSummary(string vulkanSummary)
+ {
+ // We have three ways of parsing the Vulkan version from the summary (output is a different between Windows and Linux)
+ // For now, I have decided to go with the full version number, and leave it up to the user to parse it further if needed
+ // I have left the other patterns in, in case we need them in the future
+
+ // Output on linux : 4206847 (1.3.255)
+ // Output on windows : 1.3.255
+ string pattern = @"apiVersion\s*=\s*([^\r\n]+)";
+
+ // Output on linux : 4206847
+ // Output on windows : 1.3.255
+ //string pattern = @"apiVersion\s*=\s*([\d\.]+)";
+
+ // Output on linux : 1.3.255
+ // Output on windows : 1.3.255
+ //string pattern = @"apiVersion\s*=\s*(?:\d+\s*)?(?:\(\s*)?([\d]+\.[\d]+\.[\d]+)(?:\s*\))?";
+
+ // Create a Regex object to match the pattern
+ Regex regex = new Regex(pattern);
+
+ // Match the pattern in the input string
+ Match match = regex.Match(vulkanSummary);
+
+ // If a match is found
+ if (match.Success)
+ {
+ // Return the version number
+ return match.Groups[1].Value;
+ }
+
+ // Return null if no match is found
+ return null;
+ }
+ #endregion
#region CUDA version
private static int GetCudaMajorVersion()
diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec
new file mode 100644
index 000000000..33880b037
--- /dev/null
+++ b/LLama/runtimes/build/LLamaSharp.Backend.Vulkan.nuspec
@@ -0,0 +1,31 @@
+
+
+
+ LLamaSharp.Backend.Vulkan
+ $version$
+ LLamaSharp.Backend.Vulkan, the backend for LLamaSharp
+ llama.cpp Authors
+ false
+ MIT
+ icon512.png
+ https://github.com/SciSharp/LLamaSharp
+ LLamaSharp.Backend.Vulkan is a backend for LLamaSharp to use with Vulkan.
+
+ Copyright 2023 The llama.cpp Authors. All rights reserved.
+ LLamaSharp LLama LLM GPT AI ChatBot SciSharp
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file