llamamodel: trigger CUDA OOM early so we can fall back

Signed-off-by: Jared Van Bortel <jared@nomic.ai>

.circleci/continue_config.yml

@@ -421,7 +421,7 @@ jobs:
             export PATH=$PATH:/usr/local/cuda/bin
             git submodule update --init --recursive
             cd gpt4all-backend
-            cmake -B build
+            cmake -B build -DCMAKE_BUILD_TYPE=Release
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -451,7 +451,7 @@ jobs:
           command: |
             git submodule update --init  # don't use --recursive because macOS doesn't use Kompute
             cd gpt4all-backend
-            cmake -B build -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
+            cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
             cmake --build build --parallel
       - run:
           name: Build wheel
@@ -466,13 +466,17 @@ jobs:
             - "*.whl"
 
   build-py-windows:
-    executor:
-      name: win/default
+    machine:
+      image: 'windows-server-2019-vs2019:2022.08.1'
+      resource_class: windows.large
+      shell: powershell.exe -ExecutionPolicy Bypass
     steps:
       - checkout
       - run:
-          name: Install MinGW64
-          command: choco install -y mingw --force --no-progress
+          name: Update Submodules
+          command: |
+            git submodule sync
+            git submodule update --init --recursive
       - run:
           name: Install VulkanSDK
           command: |
@@ -486,31 +490,40 @@ jobs:
       - run:
           name: Install dependencies
           command:
-            choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+            choco install -y cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
       - run:
           name: Install Python dependencies
           command: pip install setuptools wheel cmake
       - run:
           name: Build C library
           command: |
-            git submodule update --init --recursive
-            cd gpt4all-backend
-            $Env:Path += ";C:\ProgramData\mingw64\mingw64\bin"
-            $Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
+            # Visual Studio setup
+            # I would use Enter-VsDevShell but it causes cudafe++ to segfault
+            $Env:PATH += ";C:\Program Files (x86)\Windows Kits\10\bin\x64"
+            $Env:PATH += ";C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64"
+            $Env:PATH += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64"
+            $Env:LIB  = "C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64"
+            $Env:LIB += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64"
+            $Env:INCLUDE  = "C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include"
+            $Env:INCLUDE += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include"
+
+            $Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin"
             $Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
-            cmake -G "MinGW Makefiles" -B build -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DKOMPUTE_OPT_USE_BUILT_IN_VULKAN_HEADER=OFF
+            cd gpt4all-backend
+            cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Release -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
             cmake --build build --parallel
       - run:
           name: Build wheel
-          # TODO: As part of this task, we need to move mingw64 binaries into package.
-          # This is terrible and needs a more robust solution eventually.
           command: |
             cd gpt4all-bindings/python
-            cd gpt4all
-            mkdir llmodel_DO_NOT_MODIFY
-            mkdir llmodel_DO_NOT_MODIFY/build/
-            cp 'C:\ProgramData\mingw64\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/'
-            cd ..
             python setup.py bdist_wheel --plat-name=win_amd64
       - store_artifacts:
           path: gpt4all-bindings/python/dist

gpt4all-backend/llamamodel.cpp

@@ -386,12 +386,17 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     bool isEmbedding = is_embedding_arch(llama_model_arch(d_ptr->model));
     const int n_ctx_train = llama_n_ctx_train(d_ptr->model);
     if (isEmbedding) {
-        d_ptr->ctx_params.n_batch = n_ctx;
+        d_ptr->ctx_params.n_batch  = n_ctx;
+        d_ptr->ctx_params.n_ubatch = n_ctx;
     } else {
         if (n_ctx > n_ctx_train) {
             std::cerr << "warning: model was trained on only " << n_ctx_train << " context tokens ("
                       << n_ctx << " specified)\n";
         }
+
+        // GPT4All defaults to 128 tokens which is also the hardcoded maximum
+        d_ptr->ctx_params.n_batch  = LLMODEL_MAX_PROMPT_BATCH;
+        d_ptr->ctx_params.n_ubatch = LLMODEL_MAX_PROMPT_BATCH;
     }
 
     d_ptr->ctx_params.n_ctx   = n_ctx;
@@ -421,6 +426,23 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
         return false;
     }
 
+#ifdef GGML_USE_CUDA
+    if (d_ptr->model_params.n_gpu_layers > 0) {
+        try {
+            testModel(); // eagerly allocate memory
+        } catch (const std::runtime_error &e) {
+            std::cerr << "LLAMA ERROR: model test failed: " << e.what() << "\n";
+            llama_free(d_ptr->ctx);
+            d_ptr->ctx = nullptr;
+            llama_free_model(d_ptr->model);
+            d_ptr->model = nullptr;
+            d_ptr->device = -1;
+            d_ptr->deviceName.clear();
+            return false;
+        }
+    }
+#endif
+
     d_ptr->end_tokens = {llama_token_eos(d_ptr->model)};
 
     if (usingGPUDevice()) {
@@ -444,6 +466,26 @@ bool LLamaModel::loadModel(const std::string &modelPath, int n_ctx, int ngl)
     return true;
 }
 
+void LLamaModel::testModel() {
+    int n_ctx = llama_n_ctx(d_ptr->ctx);
+    int n_batch = LLMODEL_MAX_PROMPT_BATCH;
+    n_batch = std::min(n_batch, n_ctx);
+
+    // test with maximum batch size
+    PromptContext ctx;
+    ctx.n_batch = n_batch;
+    std::vector<int32_t> tokens(n_batch);
+
+    llama_set_skip_cpu(d_ptr->ctx, true);
+    if (!evalTokens(ctx, tokens))
+        throw std::runtime_error("llama_decode failed");
+    llama_set_skip_cpu(d_ptr->ctx, false);
+    llama_synchronize(d_ptr->ctx); // wait for GPU to finish
+
+    // clean up
+    llama_kv_cache_clear(d_ptr->ctx);
+}
+
 void LLamaModel::setThreadCount(int32_t n_threads) {
     d_ptr->n_threads = n_threads;
     llama_set_n_threads(d_ptr->ctx, n_threads, n_threads);
@@ -920,11 +962,11 @@ void LLamaModel::embedInternal(
         int32_t n_tokens = llama_tokenize(d_ptr->model, text.c_str(), text.length(), tokens.data(), tokens.size(), wantBOS, false);
         if (n_tokens) {
             (void)eos_token;
-            assert(useEOS == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
-            tokens.resize(n_tokens - useEOS); // erase EOS/SEP
-        } else {
-            tokens.clear();
+            assert((useEOS && wantBOS) == (eos_token != -1 && tokens[n_tokens - 1] == eos_token));
+            if (useEOS && wantBOS)
+                n_tokens--; // erase EOS/SEP
         }
+        tokens.resize(n_tokens);
     };
 
     // tokenize the texts

gpt4all-backend/llamamodel_impl.h

@@ -48,6 +48,8 @@ public:
                size_t *tokenCount = nullptr, bool doMean = true, bool atlas = false) override;
 
 private:
+    void testModel(); // used for CUDA to eagerly allocate memory
+
     std::unique_ptr<LLamaPrivate> d_ptr;
     bool m_supportsEmbedding = false;
     bool m_supportsCompletion = false;

gpt4all-backend/llmodel.h

@@ -122,7 +122,7 @@ public:
         float   top_p = 0.9f;
         float   min_p = 0.0f;
         float   temp = 0.9f;
-        int32_t n_batch = 9;
+        int32_t n_batch = 128;
         float   repeat_penalty = 1.10f;
         int32_t repeat_last_n = 64;     // last n tokens to penalize
         float   contextErase = 0.75f;   // percent of context to erase if we exceed the context window

gpt4all-chat/CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 
 set(APP_VERSION_MAJOR 2)
 set(APP_VERSION_MINOR 8)
-set(APP_VERSION_PATCH 0)
+set(APP_VERSION_PATCH 1)
 set(APP_VERSION "${APP_VERSION_MAJOR}.${APP_VERSION_MINOR}.${APP_VERSION_PATCH}")
 
 # Include the binary directory for the generated header file

gpt4all-chat/chatllm.cpp

@@ -374,6 +374,7 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
 
                 m_llModelInfo.model->setProgressCallback([this](float progress) -> bool {
                     progress = std::max(progress, std::numeric_limits<float>::min()); // keep progress above zero
+                    progress = std::min(progress, std::nextafter(1.0f, 0.0f)); // keep progress below 100% until we are actually done
                     emit modelLoadingPercentageChanged(progress);
                     return m_shouldBeLoaded;
                 });

gpt4all-chat/database.cpp

@@ -938,7 +938,7 @@ void Database::start()
     connect(m_embLLM, &EmbeddingLLM::errorGenerated, this, &Database::handleErrorGenerated);
     m_scanTimer->callOnTimeout(this, &Database::scanQueue);
     if (!QSqlDatabase::drivers().contains("QSQLITE")) {
-        qWarning() << "ERROR: missing sqllite driver";
+        qWarning() << "ERROR: missing sqlite driver";
     } else {
         QSqlError err = initDb();
         if (err.type() != QSqlError::NoError)

gpt4all-chat/metadata/release.json

@@ -810,6 +810,43 @@
 * Jared Van Bortel (Nomic AI)
 * Adam Treat (Nomic AI)
 * Community (beta testers, bug reporters, bindings authors)
+"
+  },
+  {
+    "version": "2.8.0",
+    "notes":
+"
+<b>&mdash; What's New &mdash;</b>
+* Context Menu: Replace \"Select All\" on message with \"Copy Message\" (PR #2324)
+* Context Menu: Hide Copy/Cut when nothing is selected (PR #2324)
+* Improve speed of context switch after quickly switching between several chats (PR #2343)
+* New Chat: Always switch to the new chat when the button is clicked (PR #2330)
+* New Chat: Always scroll to the top of the list when the button is clicked (PR #2330)
+* Update to latest llama.cpp as of May 9, 2024 (PR #2310)
+* **Add support for the llama.cpp CUDA backend** (PR #2310, PR #2357)
+  * Nomic Vulkan is still used by default, but CUDA devices can now be selected in Settings
+  * When in use: Greatly improved prompt processing and generation speed on some devices
+  * When in use: GPU support for Q5\_0, Q5\_1, Q8\_0, K-quants, I-quants, and Mixtral
+* Add support for InternLM models (PR #2310)
+
+<b>&mdash; Fixes &mdash;</b>
+* Do not allow sending a message while the LLM is responding (PR #2323)
+* Fix poor quality of generated chat titles with many models (PR #2322)
+* Set the window icon correctly on Windows (PR #2321)
+* Fix a few memory leaks (PR #2328, PR #2348, PR #2310)
+* Do not crash if a model file has no architecture key (PR #2346)
+* Fix several instances of model loading progress displaying incorrectly (PR #2337, PR #2343)
+* New Chat: Fix the new chat being scrolled above the top of the list on startup (PR #2330)
+* macOS: Show a \"Metal\" device option, and actually use the CPU when \"CPU\" is selected (PR #2310)
+* Remove unsupported Mamba, Persimmon, and PLaMo models from the whitelist (PR #2310)
+* Fix GPT4All.desktop being created by offline installers on macOS (PR #2361)
+",
+    "contributors":
+"
+* Jared Van Bortel (Nomic AI)
+* Adam Treat (Nomic AI)
+* Tim453 (`@Tim453`)
+* Community (beta testers, bug reporters, bindings authors)
 "
   }
 ]

gpt4all-training/old-README.md

@@ -229,7 +229,7 @@ Raw Data:
   - Explorer: https://atlas.nomic.ai/map/gpt4all_data_clean
 - [GPT4All-J Dataset](https://huggingface.co/datasets/nomic-ai/gpt4all-j-prompt-generations)
    -  Explorer Indexed on Prompts: https://atlas.nomic.ai/map/gpt4all-j-prompts-curated
-   -  Exporer Indexed on Responses: https://atlas.nomic.ai/map/gpt4all-j-response-curated
+   -  Explorer Indexed on Responses: https://atlas.nomic.ai/map/gpt4all-j-response-curated
 
 We are not distributing a LLaMa 7B checkpoint.