From c6ece6a96d8b3a9076e3528189cc69a2019e62b3 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Thu, 8 Aug 2024 12:44:21 -0700 Subject: [PATCH] core: autodetect more ls params (#25044) Co-authored-by: ccurme --- .../language_models/chat_models.py | 30 +- .../__snapshots__/test_runnable.ambr | 8 +- .../runnables/test_runnable_events_v1.py | 2726 +++++++------- .../runnables/test_runnable_events_v2.py | 3173 +++++++++-------- 4 files changed, 3133 insertions(+), 2804 deletions(-) diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py index c62feac069f..a8615faae35 100644 --- a/libs/core/langchain_core/language_models/chat_models.py +++ b/libs/core/langchain_core/language_models/chat_models.py @@ -522,9 +522,37 @@ class BaseChatModel(BaseLanguageModel[BaseMessage], ABC): **kwargs: Any, ) -> LangSmithParams: """Get standard params for tracing.""" - ls_params = LangSmithParams(ls_model_type="chat") + + # get default provider from class name + default_provider = self.__class__.__name__ + if default_provider.startswith("Chat"): + default_provider = default_provider[4:].lower() + elif default_provider.endswith("Chat"): + default_provider = default_provider[:-4] + default_provider = default_provider.lower() + + ls_params = LangSmithParams(ls_provider=default_provider, ls_model_type="chat") if stop: ls_params["ls_stop"] = stop + + # model + if hasattr(self, "model") and isinstance(self.model, str): + ls_params["ls_model_name"] = self.model + elif hasattr(self, "model_name") and isinstance(self.model_name, str): + ls_params["ls_model_name"] = self.model_name + + # temperature + if "temperature" in kwargs and isinstance(kwargs["temperature"], float): + ls_params["ls_temperature"] = kwargs["temperature"] + elif hasattr(self, "temperature") and isinstance(self.temperature, float): + ls_params["ls_temperature"] = self.temperature + + # max_tokens + if "max_tokens" in kwargs and isinstance(kwargs["max_tokens"], int): + ls_params["ls_max_tokens"] = kwargs["max_tokens"] + elif hasattr(self, "max_tokens") and isinstance(self.max_tokens, int): + ls_params["ls_max_tokens"] = self.max_tokens + return ls_params def _get_llm_string(self, stop: Optional[List[str]] = None, **kwargs: Any) -> str: diff --git a/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr b/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr index 88b421db6bb..819f2676a0f 100644 --- a/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr +++ b/libs/core/tests/unit_tests/runnables/__snapshots__/test_runnable.ambr @@ -1518,7 +1518,7 @@ # --- # name: test_combining_sequences.3 list([ - Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'middle': [{'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'repr': "RunnableLambda(lambda x: {'question': x[0] + x[1]})"}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nicer assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['baz, qux'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}], 'last': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 4, 'type': 'runnable', 'data': {'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'name': 'Lambda'}}, {'id': 5, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 6, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 7, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 8, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}, {'source': 2, 'target': 3}, {'source': 3, 'target': 4}, {'source': 4, 'target': 5}, {'source': 5, 'target': 6}, {'source': 7, 'target': 8}, {'source': 6, 'target': 7}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ['baz', 'qux']}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo, bar'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo, bar', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo, bar', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002'), Run(id=UUID('00000000-0000-4000-8000-000000000003'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='foo, bar', id='00000000-0000-4000-8000-000000000004')}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:3'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000003'), Run(id=UUID('00000000-0000-4000-8000-000000000005'), name='RunnableLambda', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'repr': "RunnableLambda(lambda x: {'question': x[0] + x[1]})"}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': ['foo', 'bar']}, outputs={'question': 'foobar'}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:4'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000005'), Run(id=UUID('00000000-0000-4000-8000-000000000006'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nicer assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'foobar'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nicer assistant.'), HumanMessage(content='foobar')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:5'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000006'), Run(id=UUID('00000000-0000-4000-8000-000000000007'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['baz, qux'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['baz, qux'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nicer assistant.\nHuman: foobar']}, outputs={'generations': [[{'text': 'baz, qux', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'baz, qux', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000006-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:6'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000007'), Run(id=UUID('00000000-0000-4000-8000-000000000008'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='baz, qux', id='00000000-0000-4000-8000-000000000009')}, outputs={'output': ['baz', 'qux']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:7'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000008')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), + Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'middle': [{'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'repr': "RunnableLambda(lambda x: {'question': x[0] + x[1]})"}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nicer assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['baz, qux'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}], 'last': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 4, 'type': 'runnable', 'data': {'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'name': 'Lambda'}}, {'id': 5, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 6, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 7, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 8, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}, {'source': 2, 'target': 3}, {'source': 3, 'target': 4}, {'source': 4, 'target': 5}, {'source': 5, 'target': 6}, {'source': 7, 'target': 8}, {'source': 6, 'target': 7}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ['baz', 'qux']}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo, bar'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_provider': 'fakelistchatmodel', 'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo, bar', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo, bar', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002'), Run(id=UUID('00000000-0000-4000-8000-000000000003'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='foo, bar', id='00000000-0000-4000-8000-000000000004')}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:3'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000003'), Run(id=UUID('00000000-0000-4000-8000-000000000005'), name='RunnableLambda', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'runnables', 'base', 'RunnableLambda'], 'repr': "RunnableLambda(lambda x: {'question': x[0] + x[1]})"}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': ['foo', 'bar']}, outputs={'question': 'foobar'}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:4'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000005'), Run(id=UUID('00000000-0000-4000-8000-000000000006'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nicer assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'foobar'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nicer assistant.'), HumanMessage(content='foobar')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:5'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000006'), Run(id=UUID('00000000-0000-4000-8000-000000000007'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['baz, qux'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_provider': 'fakelistchatmodel', 'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['baz, qux'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nicer assistant.\nHuman: foobar']}, outputs={'generations': [[{'text': 'baz, qux', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'baz, qux', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000006-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:6'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000007'), Run(id=UUID('00000000-0000-4000-8000-000000000008'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='baz, qux', id='00000000-0000-4000-8000-000000000009')}, outputs={'output': ['baz', 'qux']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:7'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000008')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), ]) # --- # name: test_each @@ -2653,7 +2653,7 @@ # --- # name: test_prompt_with_chat_model.2 list([ - Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'last': {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 2, 'target': 3}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': AIMessage(content='foo', id='00000000-0000-4000-8000-000000000003')}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), + Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'last': {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 2, 'target': 3}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': AIMessage(content='foo', id='00000000-0000-4000-8000-000000000003')}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_provider': 'fakelistchatmodel', 'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), ]) # --- # name: test_prompt_with_chat_model_and_parser @@ -3033,7 +3033,7 @@ # --- # name: test_prompt_with_chat_model_and_parser.1 list([ - Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'middle': [{'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}], 'last': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 4, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}, {'source': 3, 'target': 4}, {'source': 2, 'target': 3}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo, bar'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo, bar', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo, bar', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002'), Run(id=UUID('00000000-0000-4000-8000-000000000003'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='foo, bar', id='00000000-0000-4000-8000-000000000004')}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:3'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000003')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), + Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'middle': [{'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}], 'last': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 4, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}, {'source': 3, 'target': 4}, {'source': 2, 'target': 3}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo, bar'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_provider': 'fakelistchatmodel', 'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo, bar'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo, bar', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo, bar', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002'), Run(id=UUID('00000000-0000-4000-8000-000000000003'), name='CommaSeparatedListOutputParser', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='parser', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'kwargs': {}, 'name': 'CommaSeparatedListOutputParser', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'output_parsers', 'list', 'CommaSeparatedListOutputParser'], 'name': 'CommaSeparatedListOutputParser'}}, {'id': 2, 'type': 'schema', 'data': 'CommaSeparatedListOutputParserOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'input': AIMessage(content='foo, bar', id='00000000-0000-4000-8000-000000000004')}, outputs={'output': ['foo', 'bar']}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:3'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000003')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), ]) # --- # name: test_prompt_with_chat_model_async @@ -3351,7 +3351,7 @@ # --- # name: test_prompt_with_chat_model_async.2 list([ - Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'last': {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 2, 'target': 3}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': AIMessage(content='foo', id='00000000-0000-4000-8000-000000000003')}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), + Run(id=UUID('00000000-0000-4000-8000-000000000000'), name='RunnableSequence', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='chain', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'runnable', 'RunnableSequence'], 'kwargs': {'first': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, 'last': {'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}, 'name': 'RunnableSequence', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 3, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 2, 'target': 3}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': AIMessage(content='foo', id='00000000-0000-4000-8000-000000000003')}, reference_example_id=None, parent_run_id=None, tags=[], child_runs=[Run(id=UUID('00000000-0000-4000-8000-000000000001'), name='ChatPromptTemplate', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='prompt', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={}, error=None, serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'kwargs': {'input_variables': ['question'], 'messages': [{'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'SystemMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': [], 'template': 'You are a nice assistant.', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}, {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'chat', 'HumanMessagePromptTemplate'], 'kwargs': {'prompt': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'kwargs': {'input_variables': ['question'], 'template': '{question}', 'template_format': 'f-string'}, 'name': 'PromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'prompt', 'PromptTemplate'], 'name': 'PromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'PromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}}}]}, 'name': 'ChatPromptTemplate', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'PromptInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain', 'prompts', 'chat', 'ChatPromptTemplate'], 'name': 'ChatPromptTemplate'}}, {'id': 2, 'type': 'schema', 'data': 'ChatPromptTemplateOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'question': 'What is your name?'}, outputs={'output': ChatPromptValue(messages=[SystemMessage(content='You are a nice assistant.'), HumanMessage(content='What is your name?')])}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:1'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000001'), Run(id=UUID('00000000-0000-4000-8000-000000000002'), name='FakeListChatModel', start_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), run_type='llm', end_time=FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), extra={'invocation_params': {'responses': ['foo'], '_type': 'fake-list-chat-model', 'stop': None}, 'options': {'stop': None}, 'batch_size': 1, 'metadata': {'ls_provider': 'fakelistchatmodel', 'ls_model_type': 'chat'}}, error=None, serialized={'lc': 1, 'type': 'not_implemented', 'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'repr': "FakeListChatModel(responses=['foo'])", 'name': 'FakeListChatModel', 'graph': {'nodes': [{'id': 0, 'type': 'schema', 'data': 'FakeListChatModelInput'}, {'id': 1, 'type': 'runnable', 'data': {'id': ['langchain_core', 'language_models', 'fake_chat_models', 'FakeListChatModel'], 'name': 'FakeListChatModel'}}, {'id': 2, 'type': 'schema', 'data': 'FakeListChatModelOutput'}], 'edges': [{'source': 0, 'target': 1}, {'source': 1, 'target': 2}]}}, events=[{'name': 'start', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}, {'name': 'end', 'time': FakeDatetime(2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)}], inputs={'prompts': ['System: You are a nice assistant.\nHuman: What is your name?']}, outputs={'generations': [[{'text': 'foo', 'generation_info': None, 'type': 'ChatGeneration', 'message': {'lc': 1, 'type': 'constructor', 'id': ['langchain', 'schema', 'messages', 'AIMessage'], 'kwargs': {'content': 'foo', 'type': 'ai', 'id': 'run-00000000-0000-4000-8000-000000000002-0', 'tool_calls': [], 'invalid_tool_calls': []}}}]], 'llm_output': None, 'run': None}, reference_example_id=None, parent_run_id=UUID('00000000-0000-4000-8000-000000000000'), tags=['seq:step:2'], child_runs=[], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000.20230101T000000000000Z00000000-0000-4000-8000-000000000002')], trace_id=UUID('00000000-0000-4000-8000-000000000000'), dotted_order='20230101T000000000000Z00000000-0000-4000-8000-000000000000'), ]) # --- # name: test_prompt_with_llm diff --git a/libs/core/tests/unit_tests/runnables/test_runnable_events_v1.py b/libs/core/tests/unit_tests/runnables/test_runnable_events_v1.py index c73ca1e6482..c2230f560a5 100644 --- a/libs/core/tests/unit_tests/runnables/test_runnable_events_v1.py +++ b/libs/core/tests/unit_tests/runnables/test_runnable_events_v1.py @@ -57,6 +57,26 @@ async def _collect_events(events: AsyncIterator[StreamEvent]) -> List[StreamEven return events_ +def _assert_events_equal_allow_superset_metadata(events: List, expected: List) -> None: + """Assert that the events are equal.""" + assert len(events) == len(expected) + for i, (event, expected_event) in enumerate(zip(events, expected)): + # we want to allow a superset of metadata on each + event_with_edited_metadata = { + k: ( + v + if k != "metadata" + else { + metadata_k: metadata_v + for metadata_k, metadata_v in v.items() + if metadata_k in expected_event["metadata"] + } + ) + for k, v in event.items() + } + assert event_with_edited_metadata == expected_event, f"Event {i} did not match." + + async def test_event_stream_with_simple_function_tool() -> None: """Test the event stream with a function and tool""" @@ -71,80 +91,83 @@ async def test_event_stream_with_simple_function_tool() -> None: chain = RunnableLambda(foo) | get_docs events = await _collect_events(chain.astream_events({}, version="v1")) - assert events == [ - { - "event": "on_chain_start", - "run_id": "", - "parent_ids": [], - "name": "RunnableSequence", - "tags": [], - "metadata": {}, - "data": {"input": {}}, - }, - { - "event": "on_chain_start", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {}, - }, - { - "event": "on_chain_stream", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {"chunk": {"x": 5}}, - }, - { - "event": "on_chain_end", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {"input": {}, "output": {"x": 5}}, - }, - { - "event": "on_tool_start", - "name": "get_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - "metadata": {}, - "data": {"input": {"x": 5}}, - }, - { - "event": "on_tool_end", - "name": "get_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - "metadata": {}, - "data": {"input": {"x": 5}, "output": [Document(page_content="hello")]}, - }, - { - "event": "on_chain_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - "metadata": {}, - "name": "RunnableSequence", - "data": {"chunk": [Document(page_content="hello")]}, - }, - { - "event": "on_chain_end", - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - "metadata": {}, - "data": {"output": [Document(page_content="hello")]}, - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "event": "on_chain_start", + "run_id": "", + "parent_ids": [], + "name": "RunnableSequence", + "tags": [], + "metadata": {}, + "data": {"input": {}}, + }, + { + "event": "on_chain_start", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {}, + }, + { + "event": "on_chain_stream", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {"chunk": {"x": 5}}, + }, + { + "event": "on_chain_end", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {"input": {}, "output": {"x": 5}}, + }, + { + "event": "on_tool_start", + "name": "get_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + "metadata": {}, + "data": {"input": {"x": 5}}, + }, + { + "event": "on_tool_end", + "name": "get_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + "metadata": {}, + "data": {"input": {"x": 5}, "output": [Document(page_content="hello")]}, + }, + { + "event": "on_chain_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + "metadata": {}, + "name": "RunnableSequence", + "data": {"chunk": [Document(page_content="hello")]}, + }, + { + "event": "on_chain_end", + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + "metadata": {}, + "data": {"output": [Document(page_content="hello")]}, + }, + ], + ) async def test_event_stream_with_single_lambda() -> None: @@ -157,35 +180,38 @@ async def test_event_stream_with_single_lambda() -> None: chain = RunnableLambda(func=reverse) events = await _collect_events(chain.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_triple_lambda() -> None: @@ -201,116 +227,119 @@ async def test_event_stream_with_triple_lambda() -> None: | r.with_config({"run_name": "3"}) ) events = await _collect_events(chain.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "hello"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"input": "olleh", "output": "hello"}, - "event": "on_chain_end", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "hello"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"input": "olleh", "output": "hello"}, + "event": "on_chain_end", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_triple_lambda_test_filtering() -> None: @@ -330,70 +359,76 @@ async def test_event_stream_with_triple_lambda_test_filtering() -> None: events = await _collect_events( chain.astream_events("hello", include_names=["1"], version="v1") ) - assert events == [ - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + ], + ) events = await _collect_events( chain.astream_events( "hello", include_tags=["my_tag"], exclude_names=["2"], version="v1" ) ) - assert events == [ - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + ], + ) async def test_event_stream_with_lambdas_from_lambda() -> None: @@ -403,35 +438,38 @@ async def test_event_stream_with_lambdas_from_lambda() -> None: events = await _collect_events( as_lambdas.astream_events({"question": "hello"}, version="v1") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": {"answer": "goodbye"}}, - "event": "on_chain_stream", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"answer": "goodbye"}}, - "event": "on_chain_end", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": {"answer": "goodbye"}}, + "event": "on_chain_stream", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"answer": "goodbye"}}, + "event": "on_chain_end", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_astream_events_from_model() -> None: @@ -450,53 +488,56 @@ async def test_astream_events_from_model() -> None: .bind(stop="") ) events = await _collect_events(model.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chat_model_start", - "metadata": {"a": "b"}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b"}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b"}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b"}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"output": _AnyIdAIMessageChunk(content="hello world!")}, - "event": "on_chat_model_end", - "metadata": {"a": "b"}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chat_model_start", + "metadata": {"a": "b"}, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": {"a": "b"}, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": {"a": "b"}, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": {"a": "b"}, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"output": _AnyIdAIMessageChunk(content="hello world!")}, + "event": "on_chat_model_end", + "metadata": {"a": "b"}, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + ], + ) @RunnableLambda def i_dont_stream(input: Any, config: RunnableConfig) -> Any: @@ -506,96 +547,119 @@ async def test_astream_events_from_model() -> None: return model.invoke(input, config) events = await _collect_events(i_dont_stream.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, - "event": "on_chat_model_start", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": { - "input": {"messages": [[HumanMessage(content="hello")]]}, - "output": { - "generations": [ - [ - { - "generation_info": None, - "message": _AnyIdAIMessage(content="hello world!"), - "text": "hello world!", - "type": "ChatGeneration", - } - ] - ], - "llm_output": None, - "run": None, - }, + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chat_model_end", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_stream", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_end", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": { + "input": {"messages": [[HumanMessage(content="hello")]]}, + "output": { + "generations": [ + [ + { + "generation_info": None, + "message": _AnyIdAIMessage(content="hello world!"), + "text": "hello world!", + "type": "ChatGeneration", + } + ] + ], + "llm_output": None, + "run": None, + }, + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_stream", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_end", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) @RunnableLambda async def ai_dont_stream(input: Any, config: RunnableConfig) -> Any: @@ -605,96 +669,119 @@ async def test_astream_events_from_model() -> None: return await model.ainvoke(input, config) events = await _collect_events(ai_dont_stream.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, - "event": "on_chat_model_start", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": { - "input": {"messages": [[HumanMessage(content="hello")]]}, - "output": { - "generations": [ - [ - { - "generation_info": None, - "message": _AnyIdAIMessage(content="hello world!"), - "text": "hello world!", - "type": "ChatGeneration", - } - ] - ], - "llm_output": None, - "run": None, - }, + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chat_model_end", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_stream", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_end", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": { + "input": {"messages": [[HumanMessage(content="hello")]]}, + "output": { + "generations": [ + [ + { + "generation_info": None, + "message": _AnyIdAIMessage(content="hello world!"), + "text": "hello world!", + "type": "ChatGeneration", + } + ] + ], + "llm_output": None, + "run": None, + }, + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_stream", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_end", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_simple_chain() -> None: @@ -733,183 +820,186 @@ async def test_event_stream_with_simple_chain() -> None: events = await _collect_events( chain.astream_events({"question": "hello"}, version="v1") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"input": {"question": "hello"}}, - "event": "on_prompt_start", - "metadata": {"foo": "bar"}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_template", "seq:step:1"], - }, - { - "data": { - "input": {"question": "hello"}, - "output": ChatPromptValue( - messages=[ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], }, - "event": "on_prompt_end", - "metadata": {"foo": "bar"}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_template", "seq:step:1"], - }, - { - "data": { - "input": { - "messages": [ - [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_prompt_start", + "metadata": {"foo": "bar"}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_template", "seq:step:1"], + }, + { + "data": { + "input": {"question": "hello"}, + "output": ChatPromptValue( + messages=[ SystemMessage(content="You are Cat Agent 007"), HumanMessage(content="hello"), ] - ] - } - }, - "event": "on_chat_model_start", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": { - "input": { - "messages": [ - [ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ] + ), }, - "output": { - "generations": [ - [ - { - "generation_info": None, - "message": AIMessageChunk( - content="hello world!", id="ai1" - ), - "text": "hello world!", - "type": "ChatGenerationChunk", - } + "event": "on_prompt_end", + "metadata": {"foo": "bar"}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_template", "seq:step:1"], + }, + { + "data": { + "input": { + "messages": [ + [ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] ] - ], - "llm_output": None, - "run": None, + } }, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], }, - "event": "on_chat_model_end", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", + { + "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"output": AIMessageChunk(content="hello world!", id="ai1")}, - "event": "on_chain_end", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - ] + { + "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": { + "input": { + "messages": [ + [ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] + ] + }, + "output": { + "generations": [ + [ + { + "generation_info": None, + "message": AIMessageChunk( + content="hello world!", id="ai1" + ), + "text": "hello world!", + "type": "ChatGenerationChunk", + } + ] + ], + "llm_output": None, + "run": None, + }, + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"output": AIMessageChunk(content="hello world!", id="ai1")}, + "event": "on_chain_end", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + ], + ) async def test_event_streaming_with_tools() -> None: @@ -938,131 +1028,143 @@ async def test_event_streaming_with_tools() -> None: # type ignores below because the tools don't appear to be runnables to type checkers # we can remove as soon as that's fixed events = await _collect_events(parameterless.astream_events({}, version="v1")) # type: ignore - assert events == [ - { - "data": {"input": {}}, - "event": "on_tool_start", - "metadata": {}, - "name": "parameterless", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "hello"}, - "event": "on_tool_stream", - "metadata": {}, - "name": "parameterless", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "hello"}, - "event": "on_tool_end", - "metadata": {}, - "name": "parameterless", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {}}, + "event": "on_tool_start", + "metadata": {}, + "name": "parameterless", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "hello"}, + "event": "on_tool_stream", + "metadata": {}, + "name": "parameterless", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "hello"}, + "event": "on_tool_end", + "metadata": {}, + "name": "parameterless", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events(with_callbacks.astream_events({}, version="v1")) # type: ignore - assert events == [ - { - "data": {"input": {}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "world"}, - "event": "on_tool_stream", - "metadata": {}, - "name": "with_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "world"}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "world"}, + "event": "on_tool_stream", + "metadata": {}, + "name": "with_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "world"}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events( with_parameters.astream_events({"x": 1, "y": "2"}, version="v1") # type: ignore ) - assert events == [ - { - "data": {"input": {"x": 1, "y": "2"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_parameters", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": {"x": 1, "y": "2"}}, - "event": "on_tool_stream", - "metadata": {}, - "name": "with_parameters", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"x": 1, "y": "2"}}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_parameters", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"x": 1, "y": "2"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_parameters", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": {"x": 1, "y": "2"}}, + "event": "on_tool_stream", + "metadata": {}, + "name": "with_parameters", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"x": 1, "y": "2"}}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_parameters", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events( with_parameters_and_callbacks.astream_events({"x": 1, "y": "2"}, version="v1") # type: ignore ) - assert events == [ - { - "data": {"input": {"x": 1, "y": "2"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_parameters_and_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": {"x": 1, "y": "2"}}, - "event": "on_tool_stream", - "metadata": {}, - "name": "with_parameters_and_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"x": 1, "y": "2"}}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_parameters_and_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"x": 1, "y": "2"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_parameters_and_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": {"x": 1, "y": "2"}}, + "event": "on_tool_stream", + "metadata": {}, + "name": "with_parameters_and_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"x": 1, "y": "2"}}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_parameters_and_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) class HardCodedRetriever(BaseRetriever): @@ -1091,47 +1193,54 @@ async def test_event_stream_with_retriever() -> None: events = await _collect_events( retriever.astream_events({"query": "hello"}, version="v1") ) - assert events == [ - { - "data": { - "input": {"query": "hello"}, + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": { + "input": {"query": "hello"}, + }, + "event": "on_retriever_start", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_start", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "chunk": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ] + { + "data": { + "chunk": [ + Document(page_content="hello world!", metadata={"foo": "bar"}), + Document( + page_content="goodbye world!", metadata={"food": "spare"} + ), + ] + }, + "event": "on_retriever_stream", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_stream", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "output": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ], + { + "data": { + "output": [ + Document(page_content="hello world!", metadata={"foo": "bar"}), + Document( + page_content="goodbye world!", metadata={"food": "spare"} + ), + ], + }, + "event": "on_retriever_end", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_end", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + ], + ) async def test_event_stream_with_retriever_and_formatter() -> None: @@ -1155,96 +1264,104 @@ async def test_event_stream_with_retriever_and_formatter() -> None: chain = retriever | format_docs events = await _collect_events(chain.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"query": "hello"}}, - "event": "on_retriever_start", - "metadata": {}, - "name": "Retriever", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": { - "input": {"query": "hello"}, - "output": { - "documents": [ + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": {"query": "hello"}}, + "event": "on_retriever_start", + "metadata": {}, + "name": "Retriever", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": { + "input": {"query": "hello"}, + "output": { + "documents": [ + Document( + page_content="hello world!", metadata={"foo": "bar"} + ), + Document( + page_content="goodbye world!", + metadata={"food": "spare"}, + ), + ] + }, + }, + "event": "on_retriever_end", + "metadata": {}, + "name": "Retriever", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "hello world!, goodbye world!"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "hello world!, goodbye world!"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": { + "input": [ Document(page_content="hello world!", metadata={"foo": "bar"}), Document( page_content="goodbye world!", metadata={"food": "spare"} ), - ] + ], + "output": "hello world!, goodbye world!", }, + "event": "on_chain_end", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], }, - "event": "on_retriever_end", - "metadata": {}, - "name": "Retriever", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "hello world!, goodbye world!"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "hello world!, goodbye world!"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "input": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ], - "output": "hello world!, goodbye world!", + { + "data": {"output": "hello world!, goodbye world!"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chain_end", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "hello world!, goodbye world!"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + ], + ) async def test_event_stream_on_chain_with_tool() -> None: @@ -1266,80 +1383,83 @@ async def test_event_stream_on_chain_with_tool() -> None: events = await _collect_events( chain.astream_events({"a": "hello", "b": "world"}, version="v1") ) - assert events == [ - { - "data": {"input": {"a": "hello", "b": "world"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"a": "hello", "b": "world"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "concat", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": {"a": "hello", "b": "world"}, "output": "helloworld"}, - "event": "on_tool_end", - "metadata": {}, - "name": "concat", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "dlrowolleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "dlrowolleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "helloworld", "output": "dlrowolleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "dlrowolleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"a": "hello", "b": "world"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": {"a": "hello", "b": "world"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "concat", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": {"a": "hello", "b": "world"}, "output": "helloworld"}, + "event": "on_tool_end", + "metadata": {}, + "name": "concat", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "dlrowolleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "dlrowolleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "helloworld", "output": "dlrowolleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"output": "dlrowolleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) @pytest.mark.xfail(reason="Fix order of callback invocations in RunnableSequence") @@ -1368,89 +1488,92 @@ async def test_chain_ordering() -> None: for event in events: event["tags"] = sorted(event["tags"]) - assert events == [ - { - "data": {"input": "q"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": "q", "output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "q", "output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "q"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": "q", "output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "q", "output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_retry() -> None: @@ -1481,62 +1604,65 @@ async def test_event_stream_with_retry() -> None: for event in events: event["tags"] = sorted(event["tags"]) - assert events == [ - { - "data": {"input": "q"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "success"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "fail", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"input": "q", "output": "success"}, - "event": "on_chain_end", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": "success", "output": None}, - "event": "on_chain_end", - "metadata": {}, - "name": "fail", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "q"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "success"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "fail", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"input": "q", "output": "success"}, + "event": "on_chain_end", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": "success", "output": None}, + "event": "on_chain_end", + "metadata": {}, + "name": "fail", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + ], + ) async def test_with_llm() -> None: @@ -1550,110 +1676,121 @@ async def test_with_llm() -> None: events = await _collect_events( chain.astream_events({"question": "hello"}, version="v1") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"question": "hello"}}, - "event": "on_prompt_start", - "metadata": {}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_template", "seq:step:1"], - }, - { - "data": { - "input": {"question": "hello"}, - "output": ChatPromptValue( - messages=[ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_prompt_end", - "metadata": {}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_template", "seq:step:1"], - }, - { - "data": { - "input": {"prompts": ["System: You are Cat Agent 007\n" "Human: hello"]} + { + "data": {"input": {"question": "hello"}}, + "event": "on_prompt_start", + "metadata": {}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_template", "seq:step:1"], }, - "event": "on_llm_start", - "metadata": {}, - "name": "FakeStreamingListLLM", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": { - "input": { - "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] - }, - "output": { - "generations": [ - [{"generation_info": None, "text": "abc", "type": "Generation"}] - ], - "llm_output": None, - "run": None, + { + "data": { + "input": {"question": "hello"}, + "output": ChatPromptValue( + messages=[ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] + ), }, + "event": "on_prompt_end", + "metadata": {}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_template", "seq:step:1"], }, - "event": "on_llm_end", - "metadata": {}, - "name": "FakeStreamingListLLM", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "a"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "b"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "c"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "abc"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": { + "input": { + "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] + } + }, + "event": "on_llm_start", + "metadata": {}, + "name": "FakeStreamingListLLM", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": { + "input": { + "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] + }, + "output": { + "generations": [ + [ + { + "generation_info": None, + "text": "abc", + "type": "Generation", + } + ] + ], + "llm_output": None, + "run": None, + }, + }, + "event": "on_llm_end", + "metadata": {}, + "name": "FakeStreamingListLLM", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "a"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "b"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "c"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "abc"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_runnable_each() -> None: @@ -1686,53 +1823,56 @@ async def test_events_astream_config() -> None: assert model_02.invoke("hello") == AIMessage(content="Goodbye world", id="ai2") events = await _collect_events(model_02.astream_events("hello", version="v1")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chat_model_start", - "metadata": {}, - "name": "RunnableConfigurableFields", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content="Goodbye", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {}, - "name": "RunnableConfigurableFields", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {}, - "name": "RunnableConfigurableFields", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content="world", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {}, - "name": "RunnableConfigurableFields", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": AIMessageChunk(content="Goodbye world", id="ai2")}, - "event": "on_chat_model_end", - "metadata": {}, - "name": "RunnableConfigurableFields", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chat_model_start", + "metadata": {}, + "name": "RunnableConfigurableFields", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": AIMessageChunk(content="Goodbye", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {}, + "name": "RunnableConfigurableFields", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {}, + "name": "RunnableConfigurableFields", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": AIMessageChunk(content="world", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {}, + "name": "RunnableConfigurableFields", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": AIMessageChunk(content="Goodbye world", id="ai2")}, + "event": "on_chat_model_end", + "metadata": {}, + "name": "RunnableConfigurableFields", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_runnable_with_message_history() -> None: @@ -1886,7 +2026,7 @@ async def test_sync_in_async_stream_lambdas() -> None: add_one_proxy = RunnableLambda(add_one_proxy_) # type: ignore events = await _collect_events(add_one_proxy.astream_events(1, version="v1")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) async def test_async_in_async_stream_lambdas() -> None: @@ -1906,7 +2046,7 @@ async def test_async_in_async_stream_lambdas() -> None: add_one_proxy_ = RunnableLambda(add_one_proxy) # type: ignore events = await _collect_events(add_one_proxy_.astream_events(1, version="v1")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) @pytest.mark.xfail( @@ -1931,4 +2071,4 @@ async def test_sync_in_sync_lambdas() -> None: add_one_proxy_ = RunnableLambda(add_one_proxy) events = await _collect_events(add_one_proxy_.astream_events(1, version="v1")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) diff --git a/libs/core/tests/unit_tests/runnables/test_runnable_events_v2.py b/libs/core/tests/unit_tests/runnables/test_runnable_events_v2.py index f4f83a95b0a..6132c0efb0d 100644 --- a/libs/core/tests/unit_tests/runnables/test_runnable_events_v2.py +++ b/libs/core/tests/unit_tests/runnables/test_runnable_events_v2.py @@ -50,6 +50,9 @@ from langchain_core.runnables.schema import StreamEvent from langchain_core.runnables.utils import Input, Output from langchain_core.tools import tool from langchain_core.utils.aiter import aclosing +from tests.unit_tests.runnables.test_runnable_events_v1 import ( + _assert_events_equal_allow_superset_metadata, +) from tests.unit_tests.stubs import _AnyIdAIMessage, _AnyIdAIMessageChunk @@ -106,80 +109,83 @@ async def test_event_stream_with_simple_function_tool() -> None: chain = RunnableLambda(foo) | get_docs events = await _collect_events(chain.astream_events({}, version="v2")) - assert events == [ - { - "event": "on_chain_start", - "run_id": "", - "parent_ids": [], - "name": "RunnableSequence", - "tags": [], - "metadata": {}, - "data": {"input": {}}, - }, - { - "event": "on_chain_start", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {}, - }, - { - "event": "on_chain_stream", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {"chunk": {"x": 5}}, - }, - { - "event": "on_chain_end", - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - "metadata": {}, - "data": {"input": {}, "output": {"x": 5}}, - }, - { - "event": "on_tool_start", - "name": "get_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - "metadata": {}, - "data": {"input": {"x": 5}}, - }, - { - "event": "on_tool_end", - "name": "get_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - "metadata": {}, - "data": {"input": {"x": 5}, "output": [Document(page_content="hello")]}, - }, - { - "event": "on_chain_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - "metadata": {}, - "name": "RunnableSequence", - "data": {"chunk": [Document(page_content="hello")]}, - }, - { - "event": "on_chain_end", - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - "metadata": {}, - "data": {"output": [Document(page_content="hello")]}, - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "event": "on_chain_start", + "run_id": "", + "parent_ids": [], + "name": "RunnableSequence", + "tags": [], + "metadata": {}, + "data": {"input": {}}, + }, + { + "event": "on_chain_start", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {}, + }, + { + "event": "on_chain_stream", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {"chunk": {"x": 5}}, + }, + { + "event": "on_chain_end", + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + "metadata": {}, + "data": {"input": {}, "output": {"x": 5}}, + }, + { + "event": "on_tool_start", + "name": "get_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + "metadata": {}, + "data": {"input": {"x": 5}}, + }, + { + "event": "on_tool_end", + "name": "get_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + "metadata": {}, + "data": {"input": {"x": 5}, "output": [Document(page_content="hello")]}, + }, + { + "event": "on_chain_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + "metadata": {}, + "name": "RunnableSequence", + "data": {"chunk": [Document(page_content="hello")]}, + }, + { + "event": "on_chain_end", + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + "metadata": {}, + "data": {"output": [Document(page_content="hello")]}, + }, + ], + ) async def test_event_stream_with_single_lambda() -> None: @@ -192,35 +198,38 @@ async def test_event_stream_with_single_lambda() -> None: chain = RunnableLambda(func=reverse) events = await _collect_events(chain.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_triple_lambda() -> None: @@ -236,116 +245,119 @@ async def test_event_stream_with_triple_lambda() -> None: | r.with_config({"run_name": "3"}) ) events = await _collect_events(chain.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "hello"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"input": "olleh", "output": "hello"}, - "event": "on_chain_end", - "metadata": {}, - "name": "2", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "hello", "output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:3"], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "hello"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"input": "olleh", "output": "hello"}, + "event": "on_chain_end", + "metadata": {}, + "name": "2", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "hello", "output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:3"], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_exception() -> None: @@ -381,70 +393,76 @@ async def test_event_stream_with_triple_lambda_test_filtering() -> None: events = await _collect_events( chain.astream_events("hello", include_names=["1"], version="v2") ) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "1", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "1", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + ], + ) events = await _collect_events( chain.astream_events( "hello", include_tags=["my_tag"], exclude_names=["2"], version="v2" ) ) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - { - "data": {"chunk": "olleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - { - "data": {"output": "olleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "3", - "run_id": "", - "parent_ids": [], - "tags": ["my_tag", "seq:step:3"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + { + "data": {"chunk": "olleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + { + "data": {"output": "olleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "3", + "run_id": "", + "parent_ids": [], + "tags": ["my_tag", "seq:step:3"], + }, + ], + ) async def test_event_stream_with_lambdas_from_lambda() -> None: @@ -454,35 +472,38 @@ async def test_event_stream_with_lambdas_from_lambda() -> None: events = await _collect_events( as_lambdas.astream_events({"question": "hello"}, version="v2") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": {"answer": "goodbye"}}, - "event": "on_chain_stream", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"answer": "goodbye"}}, - "event": "on_chain_end", - "metadata": {}, - "name": "my_lambda", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": {"answer": "goodbye"}}, + "event": "on_chain_stream", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"answer": "goodbye"}}, + "event": "on_chain_end", + "metadata": {}, + "name": "my_lambda", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_astream_events_from_model() -> None: @@ -501,55 +522,78 @@ async def test_astream_events_from_model() -> None: .bind(stop="") ) events = await _collect_events(model.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chat_model_start", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": { - "output": _AnyIdAIMessageChunk(content="hello world!"), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], }, - "event": "on_chat_model_end", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - ] + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": { + "output": _AnyIdAIMessageChunk(content="hello world!"), + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + ], + ) async def test_astream_with_model_in_chain() -> None: @@ -576,83 +620,106 @@ async def test_astream_with_model_in_chain() -> None: return model.invoke(input, config) events = await _collect_events(i_dont_stream.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, - "event": "on_chat_model_start", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": { - "input": {"messages": [[HumanMessage(content="hello")]]}, - "output": _AnyIdAIMessage(content="hello world!"), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chat_model_end", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_stream", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_end", - "metadata": {}, - "name": "i_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": { + "input": {"messages": [[HumanMessage(content="hello")]]}, + "output": _AnyIdAIMessage(content="hello world!"), + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_stream", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_end", + "metadata": {}, + "name": "i_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) @RunnableLambda async def ai_dont_stream(input: Any, config: RunnableConfig) -> Any: @@ -662,83 +729,106 @@ async def test_astream_with_model_in_chain() -> None: return await model.ainvoke(input, config) events = await _collect_events(ai_dont_stream.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, - "event": "on_chat_model_start", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, - "event": "on_chat_model_stream", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": { - "input": {"messages": [[HumanMessage(content="hello")]]}, - "output": _AnyIdAIMessage(content="hello world!"), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chat_model_end", - "metadata": {"a": "b", "ls_model_type": "chat", "ls_stop": ""}, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_model"], - }, - { - "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_stream", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": _AnyIdAIMessage(content="hello world!")}, - "event": "on_chain_end", - "metadata": {}, - "name": "ai_dont_stream", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": {"input": {"messages": [[HumanMessage(content="hello")]]}}, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="hello")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content=" ")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessageChunk(content="world!")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": { + "input": {"messages": [[HumanMessage(content="hello")]]}, + "output": _AnyIdAIMessage(content="hello world!"), + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_model"], + }, + { + "data": {"chunk": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_stream", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": _AnyIdAIMessage(content="hello world!")}, + "event": "on_chain_end", + "metadata": {}, + "name": "ai_dont_stream", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_simple_chain() -> None: @@ -777,168 +867,171 @@ async def test_event_stream_with_simple_chain() -> None: events = await _collect_events( chain.astream_events({"question": "hello"}, version="v2") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"input": {"question": "hello"}}, - "event": "on_prompt_start", - "metadata": {"foo": "bar"}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_template", "seq:step:1"], - }, - { - "data": { - "input": {"question": "hello"}, - "output": ChatPromptValue( - messages=[ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], }, - "event": "on_prompt_end", - "metadata": {"foo": "bar"}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_template", "seq:step:1"], - }, - { - "data": { - "input": { - "messages": [ - [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_prompt_start", + "metadata": {"foo": "bar"}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_template", "seq:step:1"], + }, + { + "data": { + "input": {"question": "hello"}, + "output": ChatPromptValue( + messages=[ SystemMessage(content="You are Cat Agent 007"), HumanMessage(content="hello"), ] - ] - } - }, - "event": "on_chat_model_start", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, - "event": "on_chat_model_stream", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", - }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, - "event": "on_chain_stream", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - { - "data": { - "input": { - "messages": [ - [ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ] + ), }, - "output": AIMessageChunk(content="hello world!", id="ai1"), + "event": "on_prompt_end", + "metadata": {"foo": "bar"}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_template", "seq:step:1"], }, - "event": "on_chat_model_end", - "metadata": { - "a": "b", - "foo": "bar", - "ls_model_type": "chat", - "ls_stop": "", + { + "data": { + "input": { + "messages": [ + [ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] + ] + } + }, + "event": "on_chat_model_start", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], }, - "name": "my_model", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain", "my_model", "seq:step:2"], - }, - { - "data": {"output": AIMessageChunk(content="hello world!", id="ai1")}, - "event": "on_chain_end", - "metadata": {"foo": "bar"}, - "name": "my_chain", - "run_id": "", - "parent_ids": [], - "tags": ["my_chain"], - }, - ] + { + "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"chunk": AIMessageChunk(content="hello", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, + "event": "on_chat_model_stream", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"chunk": AIMessageChunk(content="world!", id="ai1")}, + "event": "on_chain_stream", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + { + "data": { + "input": { + "messages": [ + [ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] + ] + }, + "output": AIMessageChunk(content="hello world!", id="ai1"), + }, + "event": "on_chat_model_end", + "metadata": { + "a": "b", + "foo": "bar", + "ls_model_type": "chat", + "ls_stop": "", + }, + "name": "my_model", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain", "my_model", "seq:step:2"], + }, + { + "data": {"output": AIMessageChunk(content="hello world!", id="ai1")}, + "event": "on_chain_end", + "metadata": {"foo": "bar"}, + "name": "my_chain", + "run_id": "", + "parent_ids": [], + "tags": ["my_chain"], + }, + ], + ) async def test_event_streaming_with_tools() -> None: @@ -967,94 +1060,106 @@ async def test_event_streaming_with_tools() -> None: # type ignores below because the tools don't appear to be runnables to type checkers # we can remove as soon as that's fixed events = await _collect_events(parameterless.astream_events({}, version="v2")) # type: ignore - assert events == [ - { - "data": {"input": {}}, - "event": "on_tool_start", - "metadata": {}, - "name": "parameterless", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "hello"}, - "event": "on_tool_end", - "metadata": {}, - "name": "parameterless", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {}}, + "event": "on_tool_start", + "metadata": {}, + "name": "parameterless", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "hello"}, + "event": "on_tool_end", + "metadata": {}, + "name": "parameterless", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events(with_callbacks.astream_events({}, version="v2")) # type: ignore - assert events == [ - { - "data": {"input": {}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "world"}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "world"}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events( with_parameters.astream_events({"x": 1, "y": "2"}, version="v2") # type: ignore ) - assert events == [ - { - "data": {"input": {"x": 1, "y": "2"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_parameters", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"x": 1, "y": "2"}}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_parameters", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"x": 1, "y": "2"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_parameters", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"x": 1, "y": "2"}}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_parameters", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) events = await _collect_events( with_parameters_and_callbacks.astream_events({"x": 1, "y": "2"}, version="v2") # type: ignore ) - assert events == [ - { - "data": {"input": {"x": 1, "y": "2"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "with_parameters_and_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": {"x": 1, "y": "2"}}, - "event": "on_tool_end", - "metadata": {}, - "name": "with_parameters_and_callbacks", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"x": 1, "y": "2"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "with_parameters_and_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": {"x": 1, "y": "2"}}, + "event": "on_tool_end", + "metadata": {}, + "name": "with_parameters_and_callbacks", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) class HardCodedRetriever(BaseRetriever): @@ -1083,33 +1188,38 @@ async def test_event_stream_with_retriever() -> None: events = await _collect_events( retriever.astream_events({"query": "hello"}, version="v2") ) - assert events == [ - { - "data": { - "input": {"query": "hello"}, + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": { + "input": {"query": "hello"}, + }, + "event": "on_retriever_start", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_start", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "output": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ] + { + "data": { + "output": [ + Document(page_content="hello world!", metadata={"foo": "bar"}), + Document( + page_content="goodbye world!", metadata={"food": "spare"} + ), + ] + }, + "event": "on_retriever_end", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_end", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + ], + ) async def test_event_stream_with_retriever_and_formatter() -> None: @@ -1133,92 +1243,99 @@ async def test_event_stream_with_retriever_and_formatter() -> None: chain = retriever | format_docs events = await _collect_events(chain.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"query": "hello"}}, - "event": "on_retriever_start", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": { - "input": {"query": "hello"}, - "output": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ], + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_retriever_end", - "metadata": {}, - "name": "HardCodedRetriever", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "hello world!, goodbye world!"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "hello world!, goodbye world!"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "input": [ - Document(page_content="hello world!", metadata={"foo": "bar"}), - Document(page_content="goodbye world!", metadata={"food": "spare"}), - ], - "output": "hello world!, goodbye world!", + { + "data": {"input": {"query": "hello"}}, + "event": "on_retriever_start", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], }, - "event": "on_chain_end", - "metadata": {}, - "name": "format_docs", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "hello world!, goodbye world!"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": { + "input": {"query": "hello"}, + "output": [ + Document(page_content="hello world!", metadata={"foo": "bar"}), + Document( + page_content="goodbye world!", metadata={"food": "spare"} + ), + ], + }, + "event": "on_retriever_end", + "metadata": {}, + "name": "HardCodedRetriever", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "hello world!, goodbye world!"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "hello world!, goodbye world!"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": { + "input": [ + Document(page_content="hello world!", metadata={"foo": "bar"}), + Document( + page_content="goodbye world!", metadata={"food": "spare"} + ), + ], + "output": "hello world!, goodbye world!", + }, + "event": "on_chain_end", + "metadata": {}, + "name": "format_docs", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"output": "hello world!, goodbye world!"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_on_chain_with_tool() -> None: @@ -1240,80 +1357,83 @@ async def test_event_stream_on_chain_with_tool() -> None: events = await _collect_events( chain.astream_events({"a": "hello", "b": "world"}, version="v2") ) - assert events == [ - { - "data": {"input": {"a": "hello", "b": "world"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"a": "hello", "b": "world"}}, - "event": "on_tool_start", - "metadata": {}, - "name": "concat", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": {"a": "hello", "b": "world"}, "output": "helloworld"}, - "event": "on_tool_end", - "metadata": {}, - "name": "concat", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "dlrowolleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "dlrowolleh"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "helloworld", "output": "dlrowolleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "reverse", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "dlrowolleh"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"a": "hello", "b": "world"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": {"a": "hello", "b": "world"}}, + "event": "on_tool_start", + "metadata": {}, + "name": "concat", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": {"a": "hello", "b": "world"}, "output": "helloworld"}, + "event": "on_tool_end", + "metadata": {}, + "name": "concat", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "dlrowolleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "dlrowolleh"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "helloworld", "output": "dlrowolleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "reverse", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"output": "dlrowolleh"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) @pytest.mark.xfail(reason="Fix order of callback invocations in RunnableSequence") @@ -1342,89 +1462,92 @@ async def test_chain_ordering() -> None: for event in events: event["tags"] = sorted(event["tags"]) - assert events == [ - { - "data": {"input": "q"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"input": "q", "output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "foo", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "q"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": "q", "output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "bar", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"output": "q"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "q"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"input": "q", "output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "foo", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "q"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"input": "q", "output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "bar", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"output": "q"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_event_stream_with_retry() -> None: @@ -1455,53 +1578,56 @@ async def test_event_stream_with_retry() -> None: for event in events: event["tags"] = sorted(event["tags"]) - assert events == [ - { - "data": {"input": "q"}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {"chunk": "success"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - { - "data": {}, - "event": "on_chain_start", - "metadata": {}, - "name": "fail", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"input": "q", "output": "success"}, - "event": "on_chain_end", - "metadata": {}, - "name": "success", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:1"], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "q"}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {"chunk": "success"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + { + "data": {}, + "event": "on_chain_start", + "metadata": {}, + "name": "fail", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"input": "q", "output": "success"}, + "event": "on_chain_end", + "metadata": {}, + "name": "success", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:1"], + }, + ], + ) async def test_with_llm() -> None: @@ -1515,109 +1641,120 @@ async def test_with_llm() -> None: events = await _collect_events( chain.astream_events({"question": "hello"}, version="v2") ) - assert events == [ - { - "data": {"input": {"question": "hello"}}, - "event": "on_chain_start", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"input": {"question": "hello"}}, - "event": "on_prompt_start", - "metadata": {}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_template", "seq:step:1"], - }, - { - "data": { - "input": {"question": "hello"}, - "output": ChatPromptValue( - messages=[ - SystemMessage(content="You are Cat Agent 007"), - HumanMessage(content="hello"), - ] - ), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"question": "hello"}}, + "event": "on_chain_start", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_prompt_end", - "metadata": {}, - "name": "my_template", - "run_id": "", - "parent_ids": [], - "tags": ["my_template", "seq:step:1"], - }, - { - "data": { - "input": {"prompts": ["System: You are Cat Agent 007\n" "Human: hello"]} + { + "data": {"input": {"question": "hello"}}, + "event": "on_prompt_start", + "metadata": {}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_template", "seq:step:1"], }, - "event": "on_llm_start", - "metadata": {}, - "name": "FakeStreamingListLLM", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": { - "input": { - "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] - }, - "output": { - "generations": [ - [{"generation_info": None, "text": "abc", "type": "Generation"}] - ], - "llm_output": None, + { + "data": { + "input": {"question": "hello"}, + "output": ChatPromptValue( + messages=[ + SystemMessage(content="You are Cat Agent 007"), + HumanMessage(content="hello"), + ] + ), }, + "event": "on_prompt_end", + "metadata": {}, + "name": "my_template", + "run_id": "", + "parent_ids": [], + "tags": ["my_template", "seq:step:1"], }, - "event": "on_llm_end", - "metadata": {}, - "name": "FakeStreamingListLLM", - "run_id": "", - "parent_ids": [], - "tags": ["seq:step:2"], - }, - { - "data": {"chunk": "a"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "b"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "c"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "abc"}, - "event": "on_chain_end", - "metadata": {}, - "name": "RunnableSequence", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": { + "input": { + "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] + } + }, + "event": "on_llm_start", + "metadata": {}, + "name": "FakeStreamingListLLM", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": { + "input": { + "prompts": ["System: You are Cat Agent 007\n" "Human: hello"] + }, + "output": { + "generations": [ + [ + { + "generation_info": None, + "text": "abc", + "type": "Generation", + } + ] + ], + "llm_output": None, + }, + }, + "event": "on_llm_end", + "metadata": {}, + "name": "FakeStreamingListLLM", + "run_id": "", + "parent_ids": [], + "tags": ["seq:step:2"], + }, + { + "data": {"chunk": "a"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "b"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "c"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "abc"}, + "event": "on_chain_end", + "metadata": {}, + "name": "RunnableSequence", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_runnable_each() -> None: @@ -1650,55 +1787,58 @@ async def test_events_astream_config() -> None: assert model_02.invoke("hello") == AIMessage(content="Goodbye world", id="ai2") events = await _collect_events(model_02.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chat_model_start", - "metadata": {"ls_model_type": "chat"}, - "name": "GenericFakeChatModel", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content="Goodbye", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {"ls_model_type": "chat"}, - "name": "GenericFakeChatModel", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content=" ", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {"ls_model_type": "chat"}, - "name": "GenericFakeChatModel", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": AIMessageChunk(content="world", id="ai2")}, - "event": "on_chat_model_stream", - "metadata": {"ls_model_type": "chat"}, - "name": "GenericFakeChatModel", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": { - "output": AIMessageChunk(content="Goodbye world", id="ai2"), + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chat_model_start", + "metadata": {"ls_model_type": "chat"}, + "name": "GenericFakeChatModel", + "run_id": "", + "parent_ids": [], + "tags": [], }, - "event": "on_chat_model_end", - "metadata": {"ls_model_type": "chat"}, - "name": "GenericFakeChatModel", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + { + "data": {"chunk": AIMessageChunk(content="Goodbye", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {"ls_model_type": "chat"}, + "name": "GenericFakeChatModel", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": AIMessageChunk(content=" ", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {"ls_model_type": "chat"}, + "name": "GenericFakeChatModel", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": AIMessageChunk(content="world", id="ai2")}, + "event": "on_chat_model_stream", + "metadata": {"ls_model_type": "chat"}, + "name": "GenericFakeChatModel", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": { + "output": AIMessageChunk(content="Goodbye world", id="ai2"), + }, + "event": "on_chat_model_end", + "metadata": {"ls_model_type": "chat"}, + "name": "GenericFakeChatModel", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_runnable_with_message_history() -> None: @@ -1847,7 +1987,7 @@ async def test_sync_in_async_stream_lambdas() -> None: add_one_proxy_ = RunnableLambda(add_one_proxy) # type: ignore events = await _collect_events(add_one_proxy_.astream_events(1, version="v2")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) async def test_async_in_async_stream_lambdas() -> None: @@ -1867,7 +2007,7 @@ async def test_async_in_async_stream_lambdas() -> None: add_one_proxy_ = RunnableLambda(add_one_proxy) # type: ignore events = await _collect_events(add_one_proxy_.astream_events(1, version="v2")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) async def test_sync_in_sync_lambdas() -> None: @@ -1887,7 +2027,7 @@ async def test_sync_in_sync_lambdas() -> None: add_one_proxy_ = RunnableLambda(add_one_proxy) events = await _collect_events(add_one_proxy_.astream_events(1, version="v2")) - assert events == EXPECTED_EVENTS + _assert_events_equal_allow_superset_metadata(events, EXPECTED_EVENTS) class StreamingRunnable(Runnable[Input, Output]): @@ -1955,53 +2095,56 @@ async def test_astream_events_from_custom_runnable() -> None: chunks = [chunk async for chunk in runnable.astream(1, version="v2")] assert chunks == ["1", "2", "3"] events = await _collect_events(runnable.astream_events(1, version="v2")) - assert events == [ - { - "data": {"input": 1}, - "event": "on_chain_start", - "metadata": {}, - "name": "StreamingRunnable", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "1"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "StreamingRunnable", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "2"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "StreamingRunnable", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "3"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "StreamingRunnable", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "123"}, - "event": "on_chain_end", - "metadata": {}, - "name": "StreamingRunnable", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": 1}, + "event": "on_chain_start", + "metadata": {}, + "name": "StreamingRunnable", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "1"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "StreamingRunnable", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "2"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "StreamingRunnable", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "3"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "StreamingRunnable", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "123"}, + "event": "on_chain_end", + "metadata": {}, + "name": "StreamingRunnable", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_parent_run_id_assignment() -> None: @@ -2029,77 +2172,80 @@ async def test_parent_run_id_assignment() -> None: parent.astream_events("hello", {"run_id": bond}, version="v2"), with_nulled_ids=False, ) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "parent", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "child", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "grandchild", - "parent_ids": [ - "00000000-0000-0000-0000-000000000007", - "00000000-0000-0000-0000-000000000008", - ], - "run_id": "00000000-0000-0000-0000-000000000009", - "tags": [], - }, - { - "data": {"input": "hello", "output": "hello"}, - "event": "on_chain_end", - "metadata": {}, - "name": "grandchild", - "parent_ids": [ - "00000000-0000-0000-0000-000000000007", - "00000000-0000-0000-0000-000000000008", - ], - "run_id": "00000000-0000-0000-0000-000000000009", - "tags": [], - }, - { - "data": {"input": "hello", "output": "hello"}, - "event": "on_chain_end", - "metadata": {}, - "name": "child", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": {"chunk": "hello"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "parent", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - { - "data": {"output": "hello"}, - "event": "on_chain_end", - "metadata": {}, - "name": "parent", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "parent", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "child", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "grandchild", + "parent_ids": [ + "00000000-0000-0000-0000-000000000007", + "00000000-0000-0000-0000-000000000008", + ], + "run_id": "00000000-0000-0000-0000-000000000009", + "tags": [], + }, + { + "data": {"input": "hello", "output": "hello"}, + "event": "on_chain_end", + "metadata": {}, + "name": "grandchild", + "parent_ids": [ + "00000000-0000-0000-0000-000000000007", + "00000000-0000-0000-0000-000000000008", + ], + "run_id": "00000000-0000-0000-0000-000000000009", + "tags": [], + }, + { + "data": {"input": "hello", "output": "hello"}, + "event": "on_chain_end", + "metadata": {}, + "name": "child", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": {"chunk": "hello"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "parent", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + { + "data": {"output": "hello"}, + "event": "on_chain_end", + "metadata": {}, + "name": "parent", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + ], + ) async def test_bad_parent_ids() -> None: @@ -2125,17 +2271,20 @@ async def test_bad_parent_ids() -> None: # Includes only a partial list of events since the run ID gets duplicated # between parent and child run ID and the callback handler throws an exception. # The exception does not get bubbled up to the user. - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "parent", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - } - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "parent", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + } + ], + ) async def test_runnable_generator() -> None: @@ -2147,44 +2296,47 @@ async def test_runnable_generator() -> None: runnable: Runnable[str, str] = RunnableGenerator(transform=generator) events = await _collect_events(runnable.astream_events("hello", version="v2")) - assert events == [ - { - "data": {"input": "hello"}, - "event": "on_chain_start", - "metadata": {}, - "name": "generator", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "1"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "generator", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"chunk": "2"}, - "event": "on_chain_stream", - "metadata": {}, - "name": "generator", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - { - "data": {"output": "12"}, - "event": "on_chain_end", - "metadata": {}, - "name": "generator", - "run_id": "", - "parent_ids": [], - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": "hello"}, + "event": "on_chain_start", + "metadata": {}, + "name": "generator", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "1"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "generator", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"chunk": "2"}, + "event": "on_chain_stream", + "metadata": {}, + "name": "generator", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + { + "data": {"output": "12"}, + "event": "on_chain_end", + "metadata": {}, + "name": "generator", + "run_id": "", + "parent_ids": [], + "tags": [], + }, + ], + ) async def test_with_explicit_config() -> None: @@ -2380,53 +2532,56 @@ async def test_custom_event() -> None: ) run_id = str(uuid1) - assert events == [ - { - "data": {"input": 1}, - "event": "on_chain_start", - "metadata": {}, - "name": "foo", - "parent_ids": [], - "run_id": run_id, - "tags": [], - }, - { - "data": {"x": 1}, - "event": "on_custom_event", - "metadata": {}, - "name": "event1", - "parent_ids": [], - "run_id": run_id, - "tags": [], - }, - { - "data": "foo", - "event": "on_custom_event", - "metadata": {}, - "name": "event2", - "parent_ids": [], - "run_id": run_id, - "tags": [], - }, - { - "data": {"chunk": 2}, - "event": "on_chain_stream", - "metadata": {}, - "name": "foo", - "parent_ids": [], - "run_id": run_id, - "tags": [], - }, - { - "data": {"output": 2}, - "event": "on_chain_end", - "metadata": {}, - "name": "foo", - "parent_ids": [], - "run_id": run_id, - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": 1}, + "event": "on_chain_start", + "metadata": {}, + "name": "foo", + "parent_ids": [], + "run_id": run_id, + "tags": [], + }, + { + "data": {"x": 1}, + "event": "on_custom_event", + "metadata": {}, + "name": "event1", + "parent_ids": [], + "run_id": run_id, + "tags": [], + }, + { + "data": "foo", + "event": "on_custom_event", + "metadata": {}, + "name": "event2", + "parent_ids": [], + "run_id": run_id, + "tags": [], + }, + { + "data": {"chunk": 2}, + "event": "on_chain_stream", + "metadata": {}, + "name": "foo", + "parent_ids": [], + "run_id": run_id, + "tags": [], + }, + { + "data": {"output": 2}, + "event": "on_chain_end", + "metadata": {}, + "name": "foo", + "parent_ids": [], + "run_id": run_id, + "tags": [], + }, + ], + ) async def test_custom_event_nested() -> None: @@ -2467,71 +2622,74 @@ async def test_custom_event_nested() -> None: run_id = str(run_id) # type: ignore[assignment] child_run_id = str(child_run_id) # type: ignore[assignment] - assert events == [ - { - "data": {"input": 1}, - "event": "on_chain_start", - "metadata": {}, - "name": "bar", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - { - "data": {"input": 1}, - "event": "on_chain_start", - "metadata": {}, - "name": "foo", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": {"x": 1}, - "event": "on_custom_event", - "metadata": {}, - "name": "event1", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": "foo", - "event": "on_custom_event", - "metadata": {}, - "name": "event2", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": {"input": 1, "output": 2}, - "event": "on_chain_end", - "metadata": {}, - "name": "foo", - "parent_ids": ["00000000-0000-0000-0000-000000000007"], - "run_id": "00000000-0000-0000-0000-000000000008", - "tags": [], - }, - { - "data": {"chunk": 2}, - "event": "on_chain_stream", - "metadata": {}, - "name": "bar", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - { - "data": {"output": 2}, - "event": "on_chain_end", - "metadata": {}, - "name": "bar", - "parent_ids": [], - "run_id": "00000000-0000-0000-0000-000000000007", - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": 1}, + "event": "on_chain_start", + "metadata": {}, + "name": "bar", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + { + "data": {"input": 1}, + "event": "on_chain_start", + "metadata": {}, + "name": "foo", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": {"x": 1}, + "event": "on_custom_event", + "metadata": {}, + "name": "event1", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": "foo", + "event": "on_custom_event", + "metadata": {}, + "name": "event2", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": {"input": 1, "output": 2}, + "event": "on_chain_end", + "metadata": {}, + "name": "foo", + "parent_ids": ["00000000-0000-0000-0000-000000000007"], + "run_id": "00000000-0000-0000-0000-000000000008", + "tags": [], + }, + { + "data": {"chunk": 2}, + "event": "on_chain_stream", + "metadata": {}, + "name": "bar", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + { + "data": {"output": 2}, + "event": "on_chain_end", + "metadata": {}, + "name": "bar", + "parent_ids": [], + "run_id": "00000000-0000-0000-0000-000000000007", + "tags": [], + }, + ], + ) async def test_custom_event_root_dispatch() -> None: @@ -2566,32 +2724,35 @@ async def test_custom_event_root_dispatch_with_in_tool() -> None: events = await _collect_events( foo.astream_events({"x": 2}, version="v2") # type: ignore[attr-defined] ) - assert events == [ - { - "data": {"input": {"x": 2}}, - "event": "on_tool_start", - "metadata": {}, - "name": "foo", - "parent_ids": [], - "run_id": "", - "tags": [], - }, - { - "data": {"x": 2}, - "event": "on_custom_event", - "metadata": {}, - "name": "event1", - "parent_ids": [], - "run_id": "", - "tags": [], - }, - { - "data": {"output": 3}, - "event": "on_tool_end", - "metadata": {}, - "name": "foo", - "parent_ids": [], - "run_id": "", - "tags": [], - }, - ] + _assert_events_equal_allow_superset_metadata( + events, + [ + { + "data": {"input": {"x": 2}}, + "event": "on_tool_start", + "metadata": {}, + "name": "foo", + "parent_ids": [], + "run_id": "", + "tags": [], + }, + { + "data": {"x": 2}, + "event": "on_custom_event", + "metadata": {}, + "name": "event1", + "parent_ids": [], + "run_id": "", + "tags": [], + }, + { + "data": {"output": 3}, + "event": "on_tool_end", + "metadata": {}, + "name": "foo", + "parent_ids": [], + "run_id": "", + "tags": [], + }, + ], + )