github/langchain
1
0
Fork 0
mirror of https://github.com/hwchase17/langchain.git synced 2025-06-25 16:13:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support multimodal messages (#11320)

Browse Source 
Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
Harrison Chase 2023-11-06 15:14:18 -08:00 committed by GitHub
parent 388f248391
commit c27400efeb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 865 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
cookbook/openai_v1_cookbook.ipynb
View File

@ -20,6 +20,64 @@
"!pip install \"openai>=1\"" "!pip install \"openai>=1\""
] ]
}, },
{
"cell_type": "code",
"execution_count": 1,
"id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136",
"metadata": {},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema.messages import HumanMessage, SystemMessage"
]
},
{
"cell_type": "markdown",
"id": "fa7e7e95-90a1-4f73-98fe-10c4b4e0951b",
"metadata": {},
"source": [
"## [Vision](https://platform.openai.com/docs/guides/vision)\n",
"\n",
"OpenAI released multi-modal models, which can take a sequence of text and images as input."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "1c8c3965-d3c9-4186-b5f3-5e67855ef916",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content='The image appears to be a diagram illustrating the architecture or components of a software system')"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chat = ChatOpenAI(model=\"gpt-4-vision-preview\")\n",
"chat.invoke(\n",
" [\n",
" HumanMessage(\n",
" content=[\n",
" {\"type\": \"text\", \"text\": \"What is this image showing\"},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://python.langchain.com/assets/images/langchain_stack-da369071b058555da3d491a695651f15.jpg\",\n",
" \"detail\": \"auto\",\n",
" },\n",
" },\n",
" ]\n",
" )\n",
" ]\n",
")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "71c34763-d1e7-4b9a-a9d7-3e4cc0dfc2c4", "id": "71c34763-d1e7-4b9a-a9d7-3e4cc0dfc2c4",
@ -32,17 +90,6 @@
"Only works with certain models. " "Only works with certain models. "
] ]
}, },
{
"cell_type": "code",
"execution_count": 1,
"id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136",
"metadata": {},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema.messages import HumanMessage, SystemMessage"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 8, "execution_count": 8,

4
libs/langchain/langchain/agents/openai_functions_multi_agent/base.py
View File

@ -87,7 +87,9 @@ def _parse_ai_message(message: BaseMessage) -> Union[List[AgentAction], AgentFin
final_tools.append(_tool) final_tools.append(_tool)
return final_tools return final_tools
return AgentFinish(return_values={"output": message.content}, log=message.content) return AgentFinish(
return_values={"output": message.content}, log=str(message.content)
)
class OpenAIMultiFunctionsAgent(BaseMultiActionAgent): class OpenAIMultiFunctionsAgent(BaseMultiActionAgent):

2
libs/langchain/langchain/agents/output_parsers/openai_functions.py
View File

@ -72,7 +72,7 @@ class OpenAIFunctionsAgentOutputParser(AgentOutputParser):
) )
return AgentFinish( return AgentFinish(
return_values={"output": message.content}, log=message.content return_values={"output": message.content}, log=str(message.content)
) )
def parse_result( def parse_result(

10
libs/langchain/langchain/callbacks/infino_callback.py
View File

@ -1,5 +1,5 @@
import time import time
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional, cast
from langchain.callbacks.base import BaseCallbackHandler from langchain.callbacks.base import BaseCallbackHandler
from langchain.schema import AgentAction, AgentFinish, LLMResult from langchain.schema import AgentAction, AgentFinish, LLMResult
@ -232,7 +232,9 @@ class InfinoCallbackHandler(BaseCallbackHandler):
self.chat_openai_model_name = model_name self.chat_openai_model_name = model_name
prompt_tokens = 0 prompt_tokens = 0
for message_list in messages: for message_list in messages:
message_string = " ".join(msg.content for msg in message_list) message_string = " ".join(
cast(str, msg.content) for msg in message_list
)
num_tokens = get_num_tokens( num_tokens = get_num_tokens(
message_string, message_string,
openai_model_name=self.chat_openai_model_name, openai_model_name=self.chat_openai_model_name,
@ -249,7 +251,9 @@ class InfinoCallbackHandler(BaseCallbackHandler):
) )
# Send the prompt to infino # Send the prompt to infino
prompt = " ".join(msg.content for sublist in messages for msg in sublist) prompt = " ".join(
cast(str, msg.content) for sublist in messages for msg in sublist
)
self._send_to_infino("prompt", prompt, is_ts=False) self._send_to_infino("prompt", prompt, is_ts=False)
# Set the error flag to indicate no error (this will get overridden # Set the error flag to indicate no error (this will get overridden

10
libs/langchain/langchain/chat_loaders/utils.py
View File

@ -21,6 +21,11 @@ def merge_chat_runs_in_session(
""" """
messages: List[BaseMessage] = [] messages: List[BaseMessage] = []
for message in chat_session["messages"]: for message in chat_session["messages"]:
if not isinstance(message.content, str):
raise ValueError(
"Chat Loaders only support messages with content type string, "
f"got {message.content}"
)
if not messages: if not messages:
messages.append(deepcopy(message)) messages.append(deepcopy(message))
elif ( elif (
@ -29,6 +34,11 @@ def merge_chat_runs_in_session(
and messages[-1].additional_kwargs["sender"] and messages[-1].additional_kwargs["sender"]
== message.additional_kwargs.get("sender") == message.additional_kwargs.get("sender")
): ):
if not isinstance(messages[-1].content, str):
raise ValueError(
"Chat Loaders only support messages with content type string, "
f"got {messages[-1].content}"
)
messages[-1].content = ( messages[-1].content = (
messages[-1].content + delimiter + message.content messages[-1].content + delimiter + message.content
).strip() ).strip()

11
libs/langchain/langchain/chat_models/anthropic.py
View File

@ -1,4 +1,4 @@
from typing import Any, AsyncIterator, Dict, Iterator, List, Optional from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, cast
from langchain.callbacks.manager import ( from langchain.callbacks.manager import (
AsyncCallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun,
@ -27,14 +27,15 @@ def _convert_one_message_to_text(
human_prompt: str, human_prompt: str,
ai_prompt: str, ai_prompt: str,
) -> str: ) -> str:
content = cast(str, message.content)
if isinstance(message, ChatMessage): if isinstance(message, ChatMessage):
message_text = f"\n\n{message.role.capitalize()}: {message.content}" message_text = f"\n\n{message.role.capitalize()}: {content}"
elif isinstance(message, HumanMessage): elif isinstance(message, HumanMessage):
message_text = f"{human_prompt} {message.content}" message_text = f"{human_prompt} {content}"
elif isinstance(message, AIMessage): elif isinstance(message, AIMessage):
message_text = f"{ai_prompt} {message.content}" message_text = f"{ai_prompt} {content}"
elif isinstance(message, SystemMessage): elif isinstance(message, SystemMessage):
message_text = message.content message_text = content
else: else:
raise ValueError(f"Got unknown type {message}") raise ValueError(f"Got unknown type {message}")
return message_text return message_text

19
libs/langchain/langchain/chat_models/azureml_endpoint.py
View File

@ -1,5 +1,5 @@
import json import json
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional, cast
from langchain.callbacks.manager import CallbackManagerForLLMRun from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.chat_models.base import SimpleChatModel from langchain.chat_models.base import SimpleChatModel
@ -23,26 +23,21 @@ class LlamaContentFormatter(ContentFormatterBase):
@staticmethod @staticmethod
def _convert_message_to_dict(message: BaseMessage) -> Dict: def _convert_message_to_dict(message: BaseMessage) -> Dict:
"""Converts message to a dict according to role""" """Converts message to a dict according to role"""
content = cast(str, message.content)
if isinstance(message, HumanMessage): if isinstance(message, HumanMessage):
return { return {
"role": "user", "role": "user",
"content": ContentFormatterBase.escape_special_characters( "content": ContentFormatterBase.escape_special_characters(content),
message.content
),
} }
elif isinstance(message, AIMessage): elif isinstance(message, AIMessage):
return { return {
"role": "assistant", "role": "assistant",
"content": ContentFormatterBase.escape_special_characters( "content": ContentFormatterBase.escape_special_characters(content),
message.content
),
} }
elif isinstance(message, SystemMessage): elif isinstance(message, SystemMessage):
return { return {
"role": "system", "role": "system",
"content": ContentFormatterBase.escape_special_characters( "content": ContentFormatterBase.escape_special_characters(content),
message.content
),
} }
elif ( elif (
isinstance(message, ChatMessage) isinstance(message, ChatMessage)
@ -50,9 +45,7 @@ class LlamaContentFormatter(ContentFormatterBase):
): ):
return { return {
"role": message.role, "role": message.role,
"content": ContentFormatterBase.escape_special_characters( "content": ContentFormatterBase.escape_special_characters(content),
message.content
),
} }
else: else:
supported = ",".join( supported = ",".join(

12
libs/langchain/langchain/chat_models/baidu_qianfan_endpoint.py
View File

@ -1,15 +1,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
from typing import ( from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, cast
Any,
AsyncIterator,
Dict,
Iterator,
List,
Mapping,
Optional,
)
from langchain.callbacks.manager import ( from langchain.callbacks.manager import (
AsyncCallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun,
@ -211,7 +203,7 @@ class QianfanChatEndpoint(BaseChatModel):
for i in [i for i, m in enumerate(messages) if isinstance(m, SystemMessage)]: for i in [i for i, m in enumerate(messages) if isinstance(m, SystemMessage)]:
if "system" not in messages_dict: if "system" not in messages_dict:
messages_dict["system"] = "" messages_dict["system"] = ""
messages_dict["system"] += messages[i].content + "\n" messages_dict["system"] += cast(str, messages[i].content) + "\n"
return { return {
**messages_dict, **messages_dict,

6
libs/langchain/langchain/chat_models/base.py
View File

@ -634,7 +634,10 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):
else: else:
_stop = list(stop) _stop = list(stop)
result = self([HumanMessage(content=text)], stop=_stop, **kwargs) result = self([HumanMessage(content=text)], stop=_stop, **kwargs)
if isinstance(result.content, str):
return result.content return result.content
else:
raise ValueError("Cannot use predict when output is not a string.")
def predict_messages( def predict_messages(
self, self,
@ -659,7 +662,10 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC):
result = await self._call_async( result = await self._call_async(
[HumanMessage(content=text)], stop=_stop, **kwargs [HumanMessage(content=text)], stop=_stop, **kwargs
) )
if isinstance(result.content, str):
return result.content return result.content
else:
raise ValueError("Cannot use predict when output is not a string.")
async def apredict_messages( async def apredict_messages(
self, self,

4
libs/langchain/langchain/chat_models/google_palm.py
View File

@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, cast
from tenacity import ( from tenacity import (
before_sleep_log, before_sleep_log,
@ -114,7 +114,7 @@ def _messages_to_prompt_dict(
if isinstance(input_message, SystemMessage): if isinstance(input_message, SystemMessage):
if index != 0: if index != 0:
raise ChatGooglePalmError("System message must be first input message.") raise ChatGooglePalmError("System message must be first input message.")
context = input_message.content context = cast(str, input_message.content)
elif isinstance(input_message, HumanMessage) and input_message.example: elif isinstance(input_message, HumanMessage) and input_message.example:
if messages: if messages:
raise ChatGooglePalmError( raise ChatGooglePalmError(

3
libs/langchain/langchain/chat_models/human.py
View File

@ -58,7 +58,10 @@ def _collect_yaml_input(
if message is None: if message is None:
return HumanMessage(content="") return HumanMessage(content="")
if stop: if stop:
if isinstance(message.content, str):
message.content = enforce_stop_tokens(message.content, stop) message.content = enforce_stop_tokens(message.content, stop)
else:
raise ValueError("Cannot use when output is not a string.")
return message return message
except yaml.YAMLError: except yaml.YAMLError:
raise ValueError("Invalid YAML string entered.") raise ValueError("Invalid YAML string entered.")

7
libs/langchain/langchain/chat_models/minimax.py
View File

@ -1,6 +1,6 @@
"""Wrapper around Minimax chat models.""" """Wrapper around Minimax chat models."""
import logging import logging
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional, cast
from langchain.callbacks.manager import ( from langchain.callbacks.manager import (
AsyncCallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun,
@ -27,10 +27,11 @@ def _parse_chat_history(history: List[BaseMessage]) -> List:
"""Parse a sequence of messages into history.""" """Parse a sequence of messages into history."""
chat_history = [] chat_history = []
for message in history: for message in history:
content = cast(str, message.content)
if isinstance(message, HumanMessage): if isinstance(message, HumanMessage):
chat_history.append(_parse_message("USER", message.content)) chat_history.append(_parse_message("USER", content))
if isinstance(message, AIMessage): if isinstance(message, AIMessage):
chat_history.append(_parse_message("BOT", message.content)) chat_history.append(_parse_message("BOT", content))
return chat_history return chat_history

6
libs/langchain/langchain/chat_models/openai.py
View File

@ -316,14 +316,16 @@ class ChatOpenAI(BaseChatModel):
@property @property
def _default_params(self) -> Dict[str, Any]: def _default_params(self) -> Dict[str, Any]:
"""Get the default parameters for calling OpenAI API.""" """Get the default parameters for calling OpenAI API."""
return { params = {
"model": self.model_name, "model": self.model_name,
"max_tokens": self.max_tokens,
"stream": self.streaming, "stream": self.streaming,
"n": self.n, "n": self.n,
"temperature": self.temperature, "temperature": self.temperature,
**self.model_kwargs, **self.model_kwargs,
} }
if "vision" not in self.model_name:
params["max_tokens"] = self.max_tokens
return params
def completion_with_retry( def completion_with_retry(
self, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any self, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any

17
libs/langchain/langchain/chat_models/pai_eas_endpoint.py
View File

@ -2,7 +2,7 @@ import asyncio
import json import json
import logging import logging
from functools import partial from functools import partial
from typing import Any, AsyncIterator, Dict, List, Optional from typing import Any, AsyncIterator, Dict, List, Optional, cast
import requests import requests
@ -133,23 +133,24 @@ class PaiEasChatEndpoint(BaseChatModel):
for message in messages: for message in messages:
"""Converts message to a dict according to role""" """Converts message to a dict according to role"""
content = cast(str, message.content)
if isinstance(message, HumanMessage): if isinstance(message, HumanMessage):
user_content = user_content + [message.content] user_content = user_content + [content]
elif isinstance(message, AIMessage): elif isinstance(message, AIMessage):
assistant_content = assistant_content + [message.content] assistant_content = assistant_content + [content]
elif isinstance(message, SystemMessage): elif isinstance(message, SystemMessage):
prompt["system_prompt"] = message.content prompt["system_prompt"] = content
elif isinstance(message, ChatMessage) and message.role in [ elif isinstance(message, ChatMessage) and message.role in [
"user", "user",
"assistant", "assistant",
"system", "system",
]: ]:
if message.role == "system": if message.role == "system":
prompt["system_prompt"] = message.content prompt["system_prompt"] = content
elif message.role == "user": elif message.role == "user":
user_content = user_content + [message.content] user_content = user_content + [content]
elif message.role == "assistant": elif message.role == "assistant":
assistant_content = assistant_content + [message.content] assistant_content = assistant_content + [content]
else: else:
supported = ",".join([role for role in ["user", "assistant", "system"]]) supported = ",".join([role for role in ["user", "assistant", "system"]])
raise ValueError( raise ValueError(
@ -294,7 +295,7 @@ class PaiEasChatEndpoint(BaseChatModel):
# yield text, if any # yield text, if any
if text: if text:
if run_manager: if run_manager:
await run_manager.on_llm_new_token(content.content) await run_manager.on_llm_new_token(cast(str, content.content))
yield ChatGenerationChunk(message=content) yield ChatGenerationChunk(message=content)
# break if stop sequence found # break if stop sequence found

5
libs/langchain/langchain/chat_models/vertexai.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import logging import logging
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
from langchain.callbacks.manager import ( from langchain.callbacks.manager import (
AsyncCallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun,
@ -57,8 +57,9 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory:
vertex_messages, context = [], None vertex_messages, context = [], None
for i, message in enumerate(history): for i, message in enumerate(history):
content = cast(str, message.content)
if i == 0 and isinstance(message, SystemMessage): if i == 0 and isinstance(message, SystemMessage):
context = message.content context = content
elif isinstance(message, AIMessage): elif isinstance(message, AIMessage):
vertex_message = ChatMessage(content=message.content, author="bot") vertex_message = ChatMessage(content=message.content, author="bot")
vertex_messages.append(vertex_message) vertex_messages.append(vertex_message)

9
libs/langchain/langchain/chat_models/yandex.py
View File

@ -1,6 +1,6 @@
"""Wrapper around YandexGPT chat models.""" """Wrapper around YandexGPT chat models."""
import logging import logging
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple, cast
from langchain.callbacks.manager import ( from langchain.callbacks.manager import (
AsyncCallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun,
@ -34,12 +34,13 @@ def _parse_chat_history(history: List[BaseMessage]) -> Tuple[List[Dict[str, str]
chat_history = [] chat_history = []
instruction = "" instruction = ""
for message in history: for message in history:
content = cast(str, message.content)
if isinstance(message, HumanMessage): if isinstance(message, HumanMessage):
chat_history.append(_parse_message("user", message.content)) chat_history.append(_parse_message("user", content))
if isinstance(message, AIMessage): if isinstance(message, AIMessage):
chat_history.append(_parse_message("assistant", message.content)) chat_history.append(_parse_message("assistant", content))
if isinstance(message, SystemMessage): if isinstance(message, SystemMessage):
instruction = message.content instruction = content
return chat_history, instruction return chat_history, instruction

39
libs/langchain/langchain/schema/messages.py
View File

@ -64,7 +64,7 @@ class BaseMessage(Serializable):
Messages are the inputs and outputs of ChatModels. Messages are the inputs and outputs of ChatModels.
""" """
content: str content: Union[str, List[Union[str, Dict]]]
"""The string contents of the message.""" """The string contents of the message."""
additional_kwargs: dict = Field(default_factory=dict) additional_kwargs: dict = Field(default_factory=dict)
@ -87,6 +87,33 @@ class BaseMessage(Serializable):
return prompt + other return prompt + other
def merge_content(
first_content: Union[str, List[Union[str, Dict]]],
second_content: Union[str, List[Union[str, Dict]]],
) -> Union[str, List[Union[str, Dict]]]:
# If first chunk is a string
if isinstance(first_content, str):
# If the second chunk is also a string, then merge them naively
if isinstance(second_content, str):
return first_content + second_content
# If the second chunk is a list, add the first chunk to the start of the list
else:
return_list: List[Union[str, Dict]] = [first_content]
return return_list + second_content
# If both are lists, merge them naively
elif isinstance(second_content, List):
return first_content + second_content
# If the first content is a list, and the second content is a string
else:
# If the last element of the first content is a string
# Add the second content to the last element
if isinstance(first_content[-1], str):
return first_content[:-1] + [first_content[-1] + second_content]
else:
# Otherwise, add the second content as a new element of the list
return first_content + [second_content]
class BaseMessageChunk(BaseMessage): class BaseMessageChunk(BaseMessage):
"""A Message chunk, which can be concatenated with other Message chunks.""" """A Message chunk, which can be concatenated with other Message chunks."""
@ -121,13 +148,13 @@ class BaseMessageChunk(BaseMessage):
if isinstance(self, ChatMessageChunk): if isinstance(self, ChatMessageChunk):
return self.__class__( return self.__class__(
role=self.role, role=self.role,
content=self.content + other.content, content=merge_content(self.content, other.content),
additional_kwargs=self._merge_kwargs_dict( additional_kwargs=self._merge_kwargs_dict(
self.additional_kwargs, other.additional_kwargs self.additional_kwargs, other.additional_kwargs
), ),
) )
return self.__class__( return self.__class__(
content=self.content + other.content, content=merge_content(self.content, other.content),
additional_kwargs=self._merge_kwargs_dict( additional_kwargs=self._merge_kwargs_dict(
self.additional_kwargs, other.additional_kwargs self.additional_kwargs, other.additional_kwargs
), ),
@ -194,7 +221,7 @@ class AIMessageChunk(AIMessage, BaseMessageChunk):
return self.__class__( return self.__class__(
example=self.example, example=self.example,
content=self.content + other.content, content=merge_content(self.content, other.content),
additional_kwargs=self._merge_kwargs_dict( additional_kwargs=self._merge_kwargs_dict(
self.additional_kwargs, other.additional_kwargs self.additional_kwargs, other.additional_kwargs
), ),
@ -252,7 +279,7 @@ class FunctionMessageChunk(FunctionMessage, BaseMessageChunk):
return self.__class__( return self.__class__(
name=self.name, name=self.name,
content=self.content + other.content, content=merge_content(self.content, other.content),
additional_kwargs=self._merge_kwargs_dict( additional_kwargs=self._merge_kwargs_dict(
self.additional_kwargs, other.additional_kwargs self.additional_kwargs, other.additional_kwargs
), ),
@ -290,7 +317,7 @@ class ChatMessageChunk(ChatMessage, BaseMessageChunk):
return self.__class__( return self.__class__(
role=self.role, role=self.role,
content=self.content + other.content, content=merge_content(self.content, other.content),
additional_kwargs=self._merge_kwargs_dict( additional_kwargs=self._merge_kwargs_dict(
self.additional_kwargs, other.additional_kwargs self.additional_kwargs, other.additional_kwargs
), ),

5
libs/langchain/tests/integration_tests/chat_models/test_fireworks.py
View File

@ -1,5 +1,6 @@
"""Test ChatFireworks wrapper.""" """Test ChatFireworks wrapper."""
import sys import sys
from typing import cast
import pytest import pytest
@ -152,7 +153,7 @@ def test_fireworks_streaming_stop_words(chat: ChatFireworks) -> None:
last_token = "" last_token = ""
for token in chat.stream("I'm Pickle Rick", stop=[","]): for token in chat.stream("I'm Pickle Rick", stop=[","]):
last_token = token.content last_token = cast(str, token.content)
assert isinstance(token.content, str) assert isinstance(token.content, str)
assert last_token[-1] == "," assert last_token[-1] == ","
@ -183,6 +184,6 @@ async def test_fireworks_astream(chat: ChatFireworks) -> None:
async for token in chat.astream( async for token in chat.astream(
"Who's the best quarterback in the NFL?", stop=[","] "Who's the best quarterback in the NFL?", stop=[","]
): ):
last_token = token.content last_token = cast(str, token.content)
assert isinstance(token.content, str) assert isinstance(token.content, str)
assert last_token[-1] == "," assert last_token[-1] == ","

8
libs/langchain/tests/integration_tests/smith/evaluation/test_runner_utils.py
View File

@ -490,7 +490,13 @@ async def test_arb_func_on_kv_singleio_dataset(
) )
def my_func(x: dict) -> str: def my_func(x: dict) -> str:
return runnable.invoke(x).content content = runnable.invoke(x).content
if isinstance(content, str):
return content
else:
raise ValueError(
f"Expected message with content type string, got {content}"
)
eval_config = RunEvalConfig(evaluators=[EvaluatorType.QA, EvaluatorType.CRITERIA]) eval_config = RunEvalConfig(evaluators=[EvaluatorType.QA, EvaluatorType.CRITERIA])
await arun_on_dataset( await arun_on_dataset(

665
libs/langchain/tests/unit_tests/schema/runnable/__snapshots__/test_runnable.ambr
View File

@ -1685,9 +1685,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -1716,9 +1733,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -1791,9 +1825,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -1822,9 +1873,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -1878,9 +1946,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -1944,9 +2029,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -1975,9 +2077,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -2050,9 +2169,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -2081,9 +2217,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2137,9 +2290,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -2187,9 +2357,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2218,9 +2405,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -2249,9 +2453,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -2280,9 +2501,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2314,9 +2552,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -2355,9 +2610,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2386,9 +2658,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -2461,9 +2750,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -2492,9 +2798,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2548,9 +2871,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -2589,9 +2929,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2620,9 +2977,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -2695,9 +3069,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -2726,9 +3117,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2782,9 +3190,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -2815,9 +3240,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -2846,9 +3288,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -2921,9 +3380,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -2952,9 +3428,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -3019,9 +3512,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([
@ -3076,9 +3586,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -3107,9 +3634,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'role': dict({ 'role': dict({
'title': 'Role', 'title': 'Role',
'type': 'string', 'type': 'string',
@ -3138,9 +3682,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'name': dict({ 'name': dict({
'title': 'Name', 'title': 'Name',
'type': 'string', 'type': 'string',
@ -3169,9 +3730,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'example': dict({ 'example': dict({
'default': False, 'default': False,
'title': 'Example', 'title': 'Example',
@ -3203,9 +3781,26 @@
'type': 'object', 'type': 'object',
}), }),
'content': dict({ 'content': dict({
'title': 'Content', 'anyOf': list([
dict({
'type': 'string', 'type': 'string',
}), }),
dict({
'items': dict({
'anyOf': list([
dict({
'type': 'string',
}),
dict({
'type': 'object',
}),
]),
}),
'type': 'array',
}),
]),
'title': 'Content',
}),
'type': dict({ 'type': dict({
'default': 'system', 'default': 'system',
'enum': list([ 'enum': list([

65
libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py
View File

@ -295,7 +295,18 @@ def test_schemas(snapshot: SnapshotAssertion) -> None:
"description": "A Message from an AI.", "description": "A Message from an AI.",
"type": "object", "type": "object",
"properties": { "properties": {
"content": {"title": "Content", "type": "string"}, "content": {
"title": "Content",
"anyOf": [
{"type": "string"},
{
"type": "array",
"items": {
"anyOf": [{"type": "string"}, {"type": "object"}]
},
},
],
},
"additional_kwargs": { "additional_kwargs": {
"title": "Additional Kwargs", "title": "Additional Kwargs",
"type": "object", "type": "object",
@ -319,7 +330,18 @@ def test_schemas(snapshot: SnapshotAssertion) -> None:
"description": "A Message from a human.", "description": "A Message from a human.",
"type": "object", "type": "object",
"properties": { "properties": {
"content": {"title": "Content", "type": "string"}, "content": {
"title": "Content",
"anyOf": [
{"type": "string"},
{
"type": "array",
"items": {
"anyOf": [{"type": "string"}, {"type": "object"}]
},
},
],
},
"additional_kwargs": { "additional_kwargs": {
"title": "Additional Kwargs", "title": "Additional Kwargs",
"type": "object", "type": "object",
@ -343,7 +365,18 @@ def test_schemas(snapshot: SnapshotAssertion) -> None:
"description": "A Message that can be assigned an arbitrary speaker (i.e. role).", # noqa: E501 "description": "A Message that can be assigned an arbitrary speaker (i.e. role).", # noqa: E501
"type": "object", "type": "object",
"properties": { "properties": {
"content": {"title": "Content", "type": "string"}, "content": {
"title": "Content",
"anyOf": [
{"type": "string"},
{
"type": "array",
"items": {
"anyOf": [{"type": "string"}, {"type": "object"}]
},
},
],
},
"additional_kwargs": { "additional_kwargs": {
"title": "Additional Kwargs", "title": "Additional Kwargs",
"type": "object", "type": "object",
@ -363,7 +396,18 @@ def test_schemas(snapshot: SnapshotAssertion) -> None:
"description": "A Message for priming AI behavior, usually passed in as the first of a sequence\nof input messages.", # noqa: E501 "description": "A Message for priming AI behavior, usually passed in as the first of a sequence\nof input messages.", # noqa: E501
"type": "object", "type": "object",
"properties": { "properties": {
"content": {"title": "Content", "type": "string"}, "content": {
"title": "Content",
"anyOf": [
{"type": "string"},
{
"type": "array",
"items": {
"anyOf": [{"type": "string"}, {"type": "object"}]
},
},
],
},
"additional_kwargs": { "additional_kwargs": {
"title": "Additional Kwargs", "title": "Additional Kwargs",
"type": "object", "type": "object",
@ -382,7 +426,18 @@ def test_schemas(snapshot: SnapshotAssertion) -> None:
"description": "A Message for passing the result of executing a function back to a model.", # noqa: E501 "description": "A Message for passing the result of executing a function back to a model.", # noqa: E501
"type": "object", "type": "object",
"properties": { "properties": {
"content": {"title": "Content", "type": "string"}, "content": {
"title": "Content",
"anyOf": [
{"type": "string"},
{
"type": "array",
"items": {
"anyOf": [{"type": "string"}, {"type": "object"}]
},
},
],
},
"additional_kwargs": { "additional_kwargs": {
"title": "Additional Kwargs", "title": "Additional Kwargs",
"type": "object", "type": "object",