From 507ae62414b05630b13b366572a0e740633bb662 Mon Sep 17 00:00:00 2001
From: jcbonnet-fwd <jcbonnet@f-wd.fr>
Date: Thu, 18 Apr 2024 15:15:13 +0200
Subject: [PATCH] Add timeout parameter for better support of openailike LLM
 tools on local computer (like LM Studio). Reuse other parameters to improve
 settings of the OpenAILike object.

---
 private_gpt/components/llm/llm_component.py | 6 +++++-
 private_gpt/settings/settings.py            | 4 ++++
 settings-vllm.yaml                          | 4 ++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/private_gpt/components/llm/llm_component.py b/private_gpt/components/llm/llm_component.py
index dae997cc..3ee66aee 100644
--- a/private_gpt/components/llm/llm_component.py
+++ b/private_gpt/components/llm/llm_component.py
@@ -105,8 +105,12 @@ class LLMComponent:
                     api_key=openai_settings.api_key,
                     model=openai_settings.model,
                     is_chat_model=True,
-                    max_tokens=None,
+                    max_tokens=settings.llm.max_new_tokens,
                     api_version="",
+                    temperature=settings.llm.temperature,
+                    tokenizer=settings.llm.tokenizer,
+                    timeout=openai_settings.request_timeout,
+                    reuse_client=False,
                 )
             case "ollama":
                 try:
diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py
index 5df68114..801f6491 100644
--- a/private_gpt/settings/settings.py
+++ b/private_gpt/settings/settings.py
@@ -202,6 +202,10 @@ class OpenAISettings(BaseModel):
         "gpt-3.5-turbo",
         description="OpenAI Model to use. Example: 'gpt-4'.",
     )
+    request_timeout: float = Field(
+        120.0,
+        description="Time elapsed until openailike server times out the request. Default is 120s. Format is float. ",
+    )
 
 
 class OllamaSettings(BaseModel):
diff --git a/settings-vllm.yaml b/settings-vllm.yaml
index 5a0a68c6..1bfab6b2 100644
--- a/settings-vllm.yaml
+++ b/settings-vllm.yaml
@@ -3,6 +3,9 @@ server:
 
 llm:
   mode: openailike
+  max_new_tokens: 512
+  tokenizer: mistralai/Mistral-7B-Instruct-v0.2
+  temperature: 0.1
 
 embedding:
   mode: huggingface
@@ -15,3 +18,4 @@ openai:
   api_base: http://localhost:8000/v1
   api_key: EMPTY
   model: facebook/opt-125m
+  request_timeout: 600.0
\ No newline at end of file