github/privateGPT
1
0
Fork 1
mirror of https://github.com/imartinez/privateGPT.git synced 2025-09-05 09:11:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Adding an LLM param to fix broken generator from llamacpp (#1519)

Browse Source
This commit is contained in:
Naveen Kannan
2024-01-17 12:10:45 -05:00
committed by GitHub
parent e326126d0d
commit 869233f0e4
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
private_gpt/components/llm/llm_component.py
View File

@@ -42,7 +42,7 @@ class LLMComponent:
context_window=settings.llm.context_window,
generate_kwargs={},
# All to GPU
model_kwargs={"n_gpu_layers": -1},
model_kwargs={"n_gpu_layers": -1, "offload_kqv": True},
# transform inputs into Llama2 format
messages_to_prompt=prompt_style.messages_to_prompt,
completion_to_prompt=prompt_style.completion_to_prompt,