python: depend on offical NVIDIA CUDA packages (#2355)

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

gpt4all-backend/dlhandle.h

@@ -58,14 +58,15 @@ public:
 #include <string>
 #include <exception>
 #include <stdexcept>
+
+#define WIN32_LEAN_AND_MEAN
 #ifndef NOMINMAX
-    #define NOMINMAX
+#   define NOMINMAX
 #endif
 #include <windows.h>
 #include <libloaderapi.h>
 
 
-
 class Dlhandle {
     HMODULE chandle;
 

gpt4all-backend/llmodel.cpp

@@ -15,8 +15,16 @@
 #include <unordered_map>
 #include <vector>
 
+#ifdef _WIN32
+#   define WIN32_LEAN_AND_MEAN
+#   ifndef NOMINMAX
+#       define NOMINMAX
+#   endif
+#   include <windows.h>
+#endif
+
 #ifdef _MSC_VER
-#include <intrin.h>
+#   include <intrin.h>
 #endif
 
 #ifndef __APPLE__
@@ -85,6 +93,20 @@ static bool isImplementation(const Dlhandle &dl) {
     return dl.get<bool(uint32_t)>("is_g4a_backend_model_implementation");
 }
 
+// Add the CUDA Toolkit to the DLL search path on Windows.
+// This is necessary for chat.exe to find CUDA when started from Qt Creator.
+static void addCudaSearchPath() {
+#ifdef _WIN32
+    if (const auto *cudaPath = _wgetenv(L"CUDA_PATH")) {
+        auto libDir = std::wstring(cudaPath) + L"\\bin";
+        if (!AddDllDirectory(libDir.c_str())) {
+            auto err = GetLastError();
+            std::wcerr << L"AddDllDirectory(\"" << libDir << L"\") failed with error 0x" << std::hex << err << L"\n";
+        }
+    }
+#endif
+}
+
 const std::vector<LLModel::Implementation> &LLModel::Implementation::implementationList() {
     if (cpu_supports_avx() == 0) {
         throw std::runtime_error("CPU does not support AVX");
@@ -95,6 +117,8 @@ const std::vector<LLModel::Implementation> &LLModel::Implementation::implementat
     static auto* libs = new std::vector<Implementation>([] () {
         std::vector<Implementation> fres;
 
+        addCudaSearchPath();
+
         std::string impl_name_re = "(gptj|llamamodel-mainline)-(cpu|metal|kompute|vulkan|cuda)";
         if (cpu_supports_avx2() == 0) {
             impl_name_re += "-avxonly";

gpt4all-bindings/python/gpt4all/_pyllmodel.py

@@ -28,6 +28,27 @@ if TYPE_CHECKING:
 EmbeddingsType = TypeVar('EmbeddingsType', bound='list[Any]')
 
 
+# Find CUDA libraries from the official packages
+cuda_found = False
+if platform.system() in ('Linux', 'Windows'):
+    try:
+        from nvidia import cuda_runtime, cublas
+    except ImportError:
+        pass  # CUDA is optional
+    else:
+        if platform.system() == 'Linux':
+            cudalib   = 'lib/libcudart.so.12'
+            cublaslib = 'lib/libcublas.so.12'
+        else:  # Windows
+            cudalib   = r'bin\cudart64_12.dll'
+            cublaslib = r'bin\cublas64_12.dll'
+
+        # preload the CUDA libs so the backend can find them
+        ctypes.CDLL(os.path.join(cuda_runtime.__path__[0], cudalib), mode=ctypes.RTLD_GLOBAL)
+        ctypes.CDLL(os.path.join(cublas.__path__[0], cublaslib), mode=ctypes.RTLD_GLOBAL)
+        cuda_found = True
+
+
 # TODO: provide a config file to make this more robust
 MODEL_LIB_PATH = importlib_resources.files("gpt4all") / "llmodel_DO_NOT_MODIFY" / "build"
 
@@ -218,7 +239,16 @@ class LLModel:
         model = llmodel.llmodel_model_create2(self.model_path, backend.encode(), ctypes.byref(err))
         if model is None:
             s = err.value
-            raise RuntimeError(f"Unable to instantiate model: {'null' if s is None else s.decode()}")
+            errmsg = 'null' if s is None else s.decode()
+
+            if (
+                backend == 'cuda'
+                and not cuda_found
+                and errmsg.startswith('Could not find any implementations for backend')
+            ):
+                print('WARNING: CUDA runtime libraries not found. Try `pip install "gpt4all[cuda]"`\n', file=sys.stderr)
+
+            raise RuntimeError(f"Unable to instantiate model: {errmsg}")
         self.model: ctypes.c_void_p | None = model
 
     def __del__(self, llmodel=llmodel):
@@ -274,11 +304,12 @@ class LLModel:
 
         all_gpus = self.list_gpus()
         available_gpus = self.list_gpus(mem_required)
-        unavailable_gpus = set(all_gpus).difference(available_gpus)
+        unavailable_gpus = [g for g in all_gpus if g not in available_gpus]
 
-        error_msg = "Unable to initialize model on GPU: {!r}".format(device)
-        error_msg += "\nAvailable GPUs: {}".format(available_gpus)
-        error_msg += "\nUnavailable GPUs due to insufficient memory or features: {}".format(unavailable_gpus)
+        error_msg = (f"Unable to initialize model on GPU: {device!r}" +
+                     f"\nAvailable GPUs: {available_gpus}")
+        if unavailable_gpus:
+            error_msg += f"\nUnavailable GPUs due to insufficient memory: {unavailable_gpus}"
         raise ValueError(error_msg)
 
     def load_model(self) -> bool:

gpt4all-bindings/python/setup.py

@@ -93,7 +93,15 @@ setup(
         'typing-extensions>=4.3.0; python_version >= "3.9" and python_version < "3.11"',
     ],
     extras_require={
+        'cuda': [
+            'nvidia-cuda-runtime-cu12',
+            'nvidia-cublas-cu12',
+        ],
+        'all': [
+            'gpt4all[cuda]; platform_system == "Windows" or platform_system == "Linux"',
+        ],
         'dev': [
+            'gpt4all[all]',
             'pytest',
             'twine',
             'wheel',

gpt4all-chat/cmake/installerscript.qs

@@ -30,7 +30,7 @@ Component.prototype.createOperations = function()
                 "workingDirectory=" + targetDirectory + "/bin",
                 "iconPath=" + targetDirectory + "/gpt4all.ico",
                 "iconId=0", "description=Open GPT4All");
-        } else if (systemInfo.productType === "osx") {
+        } else if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
             var gpt4allAppPath = targetDirectory + "/bin/gpt4all.app";
             var symlinkPath = targetDirectory + "/../GPT4All.app";
             // Remove the symlink if it already exists
@@ -56,7 +56,7 @@ Component.prototype.createOperationsForArchive = function(archive)
 {
     component.createOperationsForArchive(archive);
 
-    if (systemInfo.productType === "osx") {
+    if (systemInfo.productType === "macos" || systemInfo.productType === "osx") {
         var uninstallTargetDirectory = installer.value("TargetDir");
         var symlinkPath = uninstallTargetDirectory + "/../GPT4All.app";