From 42de5168b140c01440b392055d5418272ebd8a82 Mon Sep 17 00:00:00 2001 From: ccurme Date: Tue, 23 Apr 2024 15:55:34 -0400 Subject: [PATCH 1/3] langchain: deprecate LLMChain, RetrievalQA, and ConversationalRetrievalChain (#20751) --- .../question_answering/chat_history.ipynb | 3 - .../chains/conversational_retrieval/base.py | 76 +++++++++++++++++++ libs/langchain/langchain/chains/llm.py | 23 ++++++ .../langchain/chains/retrieval_qa/base.py | 33 ++++++++ 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/docs/docs/use_cases/question_answering/chat_history.ipynb b/docs/docs/use_cases/question_answering/chat_history.ipynb index 17b46504f27..93e9a1af1b3 100644 --- a/docs/docs/use_cases/question_answering/chat_history.ipynb +++ b/docs/docs/use_cases/question_answering/chat_history.ipynb @@ -403,16 +403,13 @@ "outputs": [], "source": [ "import bs4\n", - "from langchain import hub\n", "from langchain.chains import create_history_aware_retriever, create_retrieval_chain\n", "from langchain.chains.combine_documents import create_stuff_documents_chain\n", "from langchain_chroma import Chroma\n", "from langchain_community.chat_message_histories import ChatMessageHistory\n", "from langchain_community.document_loaders import WebBaseLoader\n", "from langchain_core.chat_history import BaseChatMessageHistory\n", - "from langchain_core.output_parsers import StrOutputParser\n", "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", - "from langchain_core.runnables import RunnablePassthrough\n", "from langchain_core.runnables.history import RunnableWithMessageHistory\n", "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n", "from langchain_text_splitters import RecursiveCharacterTextSplitter\n", diff --git a/libs/langchain/langchain/chains/conversational_retrieval/base.py b/libs/langchain/langchain/chains/conversational_retrieval/base.py index d033c3e099f..bead6f6d898 100644 --- a/libs/langchain/langchain/chains/conversational_retrieval/base.py +++ b/libs/langchain/langchain/chains/conversational_retrieval/base.py @@ -7,6 +7,7 @@ from abc import abstractmethod from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from langchain_core._api import deprecated from langchain_core.callbacks import ( AsyncCallbackManagerForChainRun, CallbackManagerForChainRun, @@ -233,9 +234,84 @@ class BaseConversationalRetrievalChain(Chain): super().save(file_path) +@deprecated( + since="0.1.17", + alternative=( + "create_history_aware_retriever together with create_retrieval_chain " + "(see example in docstring)" + ), + removal="0.3.0", +) class ConversationalRetrievalChain(BaseConversationalRetrievalChain): """Chain for having a conversation based on retrieved documents. + This class is deprecated. See below for an example implementation using + `create_retrieval_chain`. Additional walkthroughs can be found at + https://python.langchain.com/docs/use_cases/question_answering/chat_history + + .. code-block:: python + + from langchain.chains import ( + create_history_aware_retriever, + create_retrieval_chain, + ) + from langchain.chains.combine_documents import create_stuff_documents_chain + from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder + from langchain_openai import ChatOpenAI + + + retriever = ... # Your retriever + + llm = ChatOpenAI() + + # Contextualize question + contextualize_q_system_prompt = ( + "Given a chat history and the latest user question " + "which might reference context in the chat history, " + "formulate a standalone question which can be understood " + "without the chat history. Do NOT answer the question, just " + "reformulate it if needed and otherwise return it as is." + ) + contextualize_q_prompt = ChatPromptTemplate.from_messages( + [ + ("system", contextualize_q_system_prompt), + MessagesPlaceholder("chat_history"), + ("human", "{input}"), + ] + ) + history_aware_retriever = create_history_aware_retriever( + llm, retriever, contextualize_q_prompt + ) + + # Answer question + qa_system_prompt = ( + "You are an assistant for question-answering tasks. Use " + "the following pieces of retrieved context to answer the " + "question. If you don't know the answer, just say that you " + "don't know. Use three sentences maximum and keep the answer " + "concise." + "\n\n" + "{context}" + ) + qa_prompt = ChatPromptTemplate.from_messages( + [ + ("system", qa_system_prompt), + MessagesPlaceholder("chat_history"), + ("human", "{input}"), + ] + ) + # Below we use create_stuff_documents_chain to feed all retrieved context + # into the LLM. Note that we can also use StuffDocumentsChain and other + # instances of BaseCombineDocumentsChain. + question_answer_chain = create_stuff_documents_chain(llm, qa_prompt) + rag_chain = create_retrieval_chain( + history_aware_retriever, question_answer_chain + ) + + # Usage: + chat_history = [] # Collect chat history here (a sequence of messages) + rag_chain.invoke({"input": query, "chat_history": chat_history}) + This chain takes in chat history (a list of messages) and new questions, and then returns an answer to that question. The algorithm for this chain consists of three parts: diff --git a/libs/langchain/langchain/chains/llm.py b/libs/langchain/langchain/chains/llm.py index f1429028405..874cfa96fad 100644 --- a/libs/langchain/langchain/chains/llm.py +++ b/libs/langchain/langchain/chains/llm.py @@ -4,6 +4,7 @@ from __future__ import annotations import warnings from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast +from langchain_core._api import deprecated from langchain_core.callbacks import ( AsyncCallbackManager, AsyncCallbackManagerForChainRun, @@ -34,9 +35,31 @@ from langchain_core.utils.input import get_colored_text from langchain.chains.base import Chain +@deprecated( + since="0.1.17", + alternative="RunnableSequence, e.g., `prompt | llm`", + removal="0.3.0", +) class LLMChain(Chain): """Chain to run queries against LLMs. + This class is deprecated. See below for an example implementation using + LangChain runnables: + + .. code-block:: python + + from langchain_core.prompts import PromptTemplate + from langchain_openai import OpenAI + + prompt_template = "Tell me a {adjective} joke" + prompt = PromptTemplate( + input_variables=["adjective"], template=prompt_template + ) + llm = OpenAI() + chain = prompt | llm + + chain.invoke("your adjective here") + Example: .. code-block:: python diff --git a/libs/langchain/langchain/chains/retrieval_qa/base.py b/libs/langchain/langchain/chains/retrieval_qa/base.py index 0a9eaafd6f1..6d030d6f11c 100644 --- a/libs/langchain/langchain/chains/retrieval_qa/base.py +++ b/libs/langchain/langchain/chains/retrieval_qa/base.py @@ -6,6 +6,7 @@ import warnings from abc import abstractmethod from typing import Any, Dict, List, Optional +from langchain_core._api import deprecated from langchain_core.callbacks import ( AsyncCallbackManagerForChainRun, CallbackManagerForChainRun, @@ -194,9 +195,41 @@ class BaseRetrievalQA(Chain): return {self.output_key: answer} +@deprecated(since="0.1.17", alternative="create_retrieval_chain", removal="0.3.0") class RetrievalQA(BaseRetrievalQA): """Chain for question-answering against an index. + This class is deprecated. See below for an example implementation using + `create_retrieval_chain`: + + .. code-block:: python + + from langchain.chains import create_retrieval_chain + from langchain.chains.combine_documents import create_stuff_documents_chain + from langchain_core.prompts import ChatPromptTemplate + from langchain_openai import ChatOpenAI + + + retriever = ... # Your retriever + llm = ChatOpenAI() + + system_prompt = ( + "Use the given context to answer the question. " + "If you don't know the answer, say you don't know. " + "Use three sentence maximum and keep the answer concise." + "\n\n{context}" + ) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", system_prompt), + ("human", "{input}"), + ] + ) + question_answer_chain = create_stuff_documents_chain(llm, prompt) + chain = create_retrieval_chain(retriever, question_answer_chain) + + chain.invoke({"input": query}) + Example: .. code-block:: python From 72f720fa3843eb92e5dc927276f55d1167c35c42 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 23 Apr 2024 16:09:14 -0400 Subject: [PATCH 2/3] langchain[major]: Remove default instantations of LLMs from VectorstoreToolkit (#20794) Remove default instantiation from vectorstore toolkit. --- .../agent_toolkits/vectorstore/toolkit.py | 3 +-- libs/langchain/langchain/chains/flare/base.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/libs/langchain/langchain/agents/agent_toolkits/vectorstore/toolkit.py b/libs/langchain/langchain/agents/agent_toolkits/vectorstore/toolkit.py index 50b60390f84..4f6004df6d7 100644 --- a/libs/langchain/langchain/agents/agent_toolkits/vectorstore/toolkit.py +++ b/libs/langchain/langchain/agents/agent_toolkits/vectorstore/toolkit.py @@ -8,10 +8,9 @@ from langchain_community.tools.vectorstore.tool import ( ) from langchain_core.language_models import BaseLanguageModel from langchain_core.pydantic_v1 import BaseModel, Field +from langchain_core.tools import BaseTool from langchain_core.vectorstores import VectorStore -from langchain.tools import BaseTool - class VectorStoreInfo(BaseModel): """Information about a VectorStore.""" diff --git a/libs/langchain/langchain/chains/flare/base.py b/libs/langchain/langchain/chains/flare/base.py index 8beb5b82e2a..8070fc12374 100644 --- a/libs/langchain/langchain/chains/flare/base.py +++ b/libs/langchain/langchain/chains/flare/base.py @@ -5,7 +5,6 @@ from abc import abstractmethod from typing import Any, Dict, List, Optional, Sequence, Tuple import numpy as np -from langchain_community.llms.openai import OpenAI from langchain_core.callbacks import ( CallbackManagerForChainRun, ) @@ -56,11 +55,7 @@ class _ResponseChain(LLMChain): class _OpenAIResponseChain(_ResponseChain): """Chain that generates responses from user input and context.""" - llm: OpenAI = Field( - default_factory=lambda: OpenAI( - max_tokens=32, model_kwargs={"logprobs": 1}, temperature=0 - ) - ) + llm: BaseLanguageModel def _extract_tokens_and_log_probs( self, generations: List[Generation] @@ -118,7 +113,7 @@ class FlareChain(Chain): question_generator_chain: QuestionGeneratorChain """Chain that generates questions from uncertain spans.""" - response_chain: _ResponseChain = Field(default_factory=_OpenAIResponseChain) + response_chain: _ResponseChain """Chain that generates responses from user input and context.""" output_parser: FinishedOutputParser = Field(default_factory=FinishedOutputParser) """Parser that determines whether the chain is finished.""" @@ -255,6 +250,14 @@ class FlareChain(Chain): Returns: FlareChain class with the given language model. """ + try: + from langchain_openai import OpenAI + except ImportError: + raise ImportError( + "OpenAI is required for FlareChain. " + "Please install langchain-openai." + "pip install langchain-openai" + ) question_gen_chain = QuestionGeneratorChain(llm=llm) response_llm = OpenAI( max_tokens=max_generation_len, model_kwargs={"logprobs": 1}, temperature=0 From a7c347ab35bdf057ee0576b54010d672fc3b7db0 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 23 Apr 2024 16:09:32 -0400 Subject: [PATCH 3/3] langchain[patch]: Update evaluation logic that instantiates a default LLM (#20760) Favor langchain_openai over langchain_community for evaluation logic. --------- Co-authored-by: ccurme --- .../langchain/evaluation/comparison/eval_chain.py | 8 ++------ .../langchain/evaluation/criteria/eval_chain.py | 12 ++++++------ libs/langchain/langchain/evaluation/loading.py | 15 ++++++++++++++- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py index b0fd1e27a27..2bcf5632612 100644 --- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py +++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py @@ -5,8 +5,6 @@ import logging import re from typing import Any, Dict, List, Optional, Union -from langchain_community.chat_models.azure_openai import AzureChatOpenAI -from langchain_community.chat_models.openai import ChatOpenAI from langchain_core.callbacks.manager import Callbacks from langchain_core.language_models import BaseLanguageModel from langchain_core.output_parsers import BaseOutputParser @@ -254,10 +252,8 @@ class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain): ValueError: If the input variables are not as expected. """ - if not ( - isinstance(llm, (ChatOpenAI, AzureChatOpenAI)) - and llm.model_name.startswith("gpt-4") - ): + # Check if the model is GPT-4 if not raise a warning + if not hasattr(llm, "model_name") or not llm.model_name.startswith("gpt-4"): logger.warning( "This chain was only tested with GPT-4. \ Performance may be significantly worse with other models." diff --git a/libs/langchain/langchain/evaluation/criteria/eval_chain.py b/libs/langchain/langchain/evaluation/criteria/eval_chain.py index 4a18aa081b5..9df38531958 100644 --- a/libs/langchain/langchain/evaluation/criteria/eval_chain.py +++ b/libs/langchain/langchain/evaluation/criteria/eval_chain.py @@ -193,7 +193,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): Examples -------- - >>> from langchain_community.chat_models import ChatAnthropic + >>> from langchain_anthropic import ChatAnthropic >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = ChatAnthropic(temperature=0) >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"} @@ -205,7 +205,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): 'score': 0, } - >>> from langchain_community.chat_models import ChatOpenAI + >>> from langchain_openai import ChatOpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = ChatOpenAI(model="gpt-4", temperature=0) >>> criteria = "correctness" @@ -344,7 +344,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = OpenAI() >>> criteria = { @@ -432,7 +432,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = OpenAI() >>> criteria = "conciseness" @@ -487,7 +487,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = OpenAI() >>> criteria = "conciseness" @@ -568,7 +568,7 @@ class LabeledCriteriaEvalChain(CriteriaEvalChain): Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = OpenAI() >>> criteria = { diff --git a/libs/langchain/langchain/evaluation/loading.py b/libs/langchain/langchain/evaluation/loading.py index fbb17325854..27a8f348e48 100644 --- a/libs/langchain/langchain/evaluation/loading.py +++ b/libs/langchain/langchain/evaluation/loading.py @@ -1,7 +1,6 @@ """Loading datasets and evaluators.""" from typing import Any, Dict, List, Optional, Sequence, Type, Union -from langchain_community.chat_models.openai import ChatOpenAI from langchain_core.language_models import BaseLanguageModel from langchain.chains.base import Chain @@ -131,6 +130,20 @@ def load_evaluator( evaluator_cls = _EVALUATOR_MAP[evaluator] if issubclass(evaluator_cls, LLMEvalChain): try: + try: + from langchain_openai import ChatOpenAI + except ImportError: + try: + from langchain_community.chat_models.openai import ChatOpenAI + except ImportError: + raise ImportError( + "Could not import langchain_openai or fallback onto " + "langchain_community. Please install langchain_openai " + "or specify a language model explicitly. " + "It's recommended to install langchain_openai AND " + "specify a language model explicitly." + ) + llm = llm or ChatOpenAI( # type: ignore[call-arg] model="gpt-4", model_kwargs={"seed": 42}, temperature=0 )