From 9d32af72ce541393fb4700c8ed520137f7c92173 Mon Sep 17 00:00:00 2001 From: mikeFore4 <38678121+mikeFore4@users.noreply.github.com> Date: Thu, 18 Jan 2024 21:44:10 -0500 Subject: [PATCH] community[patch]: huggingface hub character removal bug fix (#16233) - **Description:** Some text-generation models on huggingface repeat the prompt in their generated response, but not all do! The tests use "gpt2" which DOES repeat the prompt and as such, the HuggingFaceHub class is hardcoded to remove the first few characters of the response (to match the len(prompt)). However, if you are using a model (such as the very popular "meta-llama/Llama-2-7b-chat-hf") that DOES NOT repeat the prompt in it's generated text, then the beginning of the generated text will be cut off. This code change fixes that bug by first checking whether the prompt is repeated in the generated response and removing it conditionally. - **Issue:** #16232 - **Dependencies:** N/A - **Twitter handle:** N/A --- libs/community/langchain_community/llms/huggingface_hub.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/llms/huggingface_hub.py b/libs/community/langchain_community/llms/huggingface_hub.py index 32facc244b0..10eb8eb7a04 100644 --- a/libs/community/langchain_community/llms/huggingface_hub.py +++ b/libs/community/langchain_community/llms/huggingface_hub.py @@ -112,8 +112,10 @@ class HuggingFaceHub(LLM): if "error" in response: raise ValueError(f"Error raised by inference API: {response['error']}") if self.client.task == "text-generation": - # Text generation return includes the starter text. - text = response[0]["generated_text"][len(prompt) :] + # Text generation sometimes return includes the starter text. + text = response[0]["generated_text"] + if text.startswith(prompt): + text = response[0]["generated_text"][len(prompt) :] elif self.client.task == "text2text-generation": text = response[0]["generated_text"] elif self.client.task == "summarization":