mirror of
https://github.com/hwchase17/langchain.git
synced 2026-02-04 00:00:34 +00:00
Compare commits
3 Commits
langchain-
...
cc/update_
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ad6715b6bb | ||
|
|
027f49c661 | ||
|
|
ce6e4bb645 |
@@ -78,4 +78,4 @@ class TestParrotMultiplyToolUnit(ToolsUnitTests):
|
||||
return {"a": 2, "b": 3}
|
||||
```
|
||||
|
||||
To learn more, check out our guide on [how to add standard tests to an integration](../../contributing/how_to/integrations/standard_tests).
|
||||
To learn more, check out our guide guides on [contributing an integration](/docs/contributing/how_to/integrations/#standard-tests).
|
||||
|
||||
@@ -7,4 +7,3 @@
|
||||
|
||||
- [**Start Here**](integrations/index.mdx): Help us integrate with your favorite vendors and tools.
|
||||
- [**Package**](integrations/package): Publish an integration package to PyPi
|
||||
- [**Standard Tests**](integrations/standard_tests): Ensure your integration passes an expected set of tests.
|
||||
|
||||
323
docs/docs/contributing/how_to/integrations/chat_models.md
Normal file
323
docs/docs/contributing/how_to/integrations/chat_models.md
Normal file
@@ -0,0 +1,323 @@
|
||||
---
|
||||
pagination_next: contributing/how_to/integrations/publish
|
||||
pagination_prev: contributing/how_to/integrations/index
|
||||
---
|
||||
# How to implement and test a chat model integration
|
||||
|
||||
This guide walks through how to implement and test a custom [chat model](/docs/concepts/chat_models) that you have developed.
|
||||
|
||||
For testing, we will rely on the `langchain-tests` dependency we added in the previous [bootstrapping guide](/docs/contributing/how_to/integrations/package).
|
||||
|
||||
## Implementation
|
||||
|
||||
Let's say you're building a simple integration package that provides a `ChatParrotLink`
|
||||
chat model integration for LangChain. Here's a simple example of what your project
|
||||
structure might look like:
|
||||
|
||||
```plaintext
|
||||
langchain-parrot-link/
|
||||
├── langchain_parrot_link/
|
||||
│ ├── __init__.py
|
||||
│ └── chat_models.py
|
||||
├── tests/
|
||||
│ ├── __init__.py
|
||||
│ └── test_chat_models.py
|
||||
├── pyproject.toml
|
||||
└── README.md
|
||||
```
|
||||
|
||||
Following the [bootstrapping guide](/docs/contributing/how_to/integrations/package),
|
||||
all of these files should already exist, except for
|
||||
`chat_models.py` and `test_chat_models.py`. We will implement these files in this guide.
|
||||
|
||||
To implement `chat_models.py`, we copy the [implementation](/docs/how_to/custom_chat_model/#implementation) from our
|
||||
[Custom Chat Model Guide](/docs/how_to/custom_chat_model). Refer to that guide for more detail.
|
||||
|
||||
<details>
|
||||
<summary>chat_models.py</summary>
|
||||
```python title="langchain_parrot_link/chat_models.py"
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
|
||||
from langchain_core.callbacks import (
|
||||
CallbackManagerForLLMRun,
|
||||
)
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.messages import (
|
||||
AIMessage,
|
||||
AIMessageChunk,
|
||||
BaseMessage,
|
||||
)
|
||||
from langchain_core.messages.ai import UsageMetadata
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from pydantic import Field
|
||||
|
||||
|
||||
class ChatParrotLink(BaseChatModel):
|
||||
"""A custom chat model that echoes the first `parrot_buffer_length` characters
|
||||
of the input.
|
||||
|
||||
When contributing an implementation to LangChain, carefully document
|
||||
the model including the initialization parameters, include
|
||||
an example of how to initialize the model and include any relevant
|
||||
links to the underlying models documentation or API.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model = ChatParrotLink(parrot_buffer_length=2, model="bird-brain-001")
|
||||
result = model.invoke([HumanMessage(content="hello")])
|
||||
result = model.batch([[HumanMessage(content="hello")],
|
||||
[HumanMessage(content="world")]])
|
||||
"""
|
||||
|
||||
model_name: str = Field(alias="model")
|
||||
"""The name of the model"""
|
||||
parrot_buffer_length: int
|
||||
"""The number of characters from the last message of the prompt to be echoed."""
|
||||
temperature: Optional[float] = None
|
||||
max_tokens: Optional[int] = None
|
||||
timeout: Optional[int] = None
|
||||
stop: Optional[List[str]] = None
|
||||
max_retries: int = 2
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
messages: List[BaseMessage],
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any,
|
||||
) -> ChatResult:
|
||||
"""Override the _generate method to implement the chat model logic.
|
||||
|
||||
This can be a call to an API, a call to a local model, or any other
|
||||
implementation that generates a response to the input prompt.
|
||||
|
||||
Args:
|
||||
messages: the prompt composed of a list of messages.
|
||||
stop: a list of strings on which the model should stop generating.
|
||||
If generation stops due to a stop token, the stop token itself
|
||||
SHOULD BE INCLUDED as part of the output. This is not enforced
|
||||
across models right now, but it's a good practice to follow since
|
||||
it makes it much easier to parse the output of the model
|
||||
downstream and understand why generation stopped.
|
||||
run_manager: A run manager with callbacks for the LLM.
|
||||
"""
|
||||
# Replace this with actual logic to generate a response from a list
|
||||
# of messages.
|
||||
last_message = messages[-1]
|
||||
tokens = last_message.content[: self.parrot_buffer_length]
|
||||
ct_input_tokens = sum(len(message.content) for message in messages)
|
||||
ct_output_tokens = len(tokens)
|
||||
message = AIMessage(
|
||||
content=tokens,
|
||||
additional_kwargs={}, # Used to add additional payload to the message
|
||||
response_metadata={ # Use for response metadata
|
||||
"time_in_seconds": 3,
|
||||
},
|
||||
usage_metadata={
|
||||
"input_tokens": ct_input_tokens,
|
||||
"output_tokens": ct_output_tokens,
|
||||
"total_tokens": ct_input_tokens + ct_output_tokens,
|
||||
},
|
||||
)
|
||||
##
|
||||
|
||||
generation = ChatGeneration(message=message)
|
||||
return ChatResult(generations=[generation])
|
||||
|
||||
def _stream(
|
||||
self,
|
||||
messages: List[BaseMessage],
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[ChatGenerationChunk]:
|
||||
"""Stream the output of the model.
|
||||
|
||||
This method should be implemented if the model can generate output
|
||||
in a streaming fashion. If the model does not support streaming,
|
||||
do not implement it. In that case streaming requests will be automatically
|
||||
handled by the _generate method.
|
||||
|
||||
Args:
|
||||
messages: the prompt composed of a list of messages.
|
||||
stop: a list of strings on which the model should stop generating.
|
||||
If generation stops due to a stop token, the stop token itself
|
||||
SHOULD BE INCLUDED as part of the output. This is not enforced
|
||||
across models right now, but it's a good practice to follow since
|
||||
it makes it much easier to parse the output of the model
|
||||
downstream and understand why generation stopped.
|
||||
run_manager: A run manager with callbacks for the LLM.
|
||||
"""
|
||||
last_message = messages[-1]
|
||||
tokens = str(last_message.content[: self.parrot_buffer_length])
|
||||
ct_input_tokens = sum(len(message.content) for message in messages)
|
||||
|
||||
for token in tokens:
|
||||
usage_metadata = UsageMetadata(
|
||||
{
|
||||
"input_tokens": ct_input_tokens,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": ct_input_tokens + 1,
|
||||
}
|
||||
)
|
||||
ct_input_tokens = 0
|
||||
chunk = ChatGenerationChunk(
|
||||
message=AIMessageChunk(content=token, usage_metadata=usage_metadata)
|
||||
)
|
||||
|
||||
if run_manager:
|
||||
# This is optional in newer versions of LangChain
|
||||
# The on_llm_new_token will be called automatically
|
||||
run_manager.on_llm_new_token(token, chunk=chunk)
|
||||
|
||||
yield chunk
|
||||
|
||||
# Let's add some other information (e.g., response metadata)
|
||||
chunk = ChatGenerationChunk(
|
||||
message=AIMessageChunk(content="", response_metadata={"time_in_sec": 3})
|
||||
)
|
||||
if run_manager:
|
||||
# This is optional in newer versions of LangChain
|
||||
# The on_llm_new_token will be called automatically
|
||||
run_manager.on_llm_new_token(token, chunk=chunk)
|
||||
yield chunk
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
"""Get the type of language model used by this chat model."""
|
||||
return "echoing-chat-model-advanced"
|
||||
|
||||
@property
|
||||
def _identifying_params(self) -> Dict[str, Any]:
|
||||
"""Return a dictionary of identifying parameters.
|
||||
|
||||
This information is used by the LangChain callback system, which
|
||||
is used for tracing purposes make it possible to monitor LLMs.
|
||||
"""
|
||||
return {
|
||||
# The model name allows users to specify custom token counting
|
||||
# rules in LLM monitoring applications (e.g., in LangSmith users
|
||||
# can provide per token pricing for their model and monitor
|
||||
# costs for the given LLM.)
|
||||
"model_name": self.model_name,
|
||||
}
|
||||
```
|
||||
</details>
|
||||
|
||||
## Testing
|
||||
|
||||
To implement our test files, we will subclass test classes from the `langchain_tests` package. These test classes contain the tests that will be run. We will just need to configure what model is tested, what parameters it is tested with, and specify any tests that should be skipped.
|
||||
|
||||
### Setup
|
||||
|
||||
First we need to install certain dependencies. These include:
|
||||
|
||||
- `pytest`: For running tests
|
||||
- `pytest-socket`: For running unit tests
|
||||
- `pytest-asyncio`: For testing async functionality
|
||||
- `langchain-tests`: For importing standard tests
|
||||
- `langchain-core`: This should already be installed, but is needed to define our integration.
|
||||
|
||||
If you followed the previous [bootstrapping guide](/docs/contributing/how_to/integrations/package/), these should already be installed.
|
||||
|
||||
### Add and configure standard tests
|
||||
There are two namespaces in the langchain-tests package:
|
||||
|
||||
[unit tests](../../../concepts/testing.mdx#unit-tests) (`langchain_tests.unit_tests`): designed to be used to test the component in isolation and without access to external services
|
||||
[integration tests](../../../concepts/testing.mdx#integration-tests) (`langchain_tests.integration_tests`): designed to be used to test the component with access to external services (in particular, the external service that the component is designed to interact with).
|
||||
|
||||
Both types of tests are implemented as [pytest class-based test suites](https://docs.pytest.org/en/7.1.x/getting-started.html#group-multiple-tests-in-a-class).
|
||||
|
||||
By subclassing the base classes for each type of standard test (see below), you get all of the standard tests for that type, and you can override the properties that the test suite uses to configure the tests.
|
||||
|
||||
Here's how you would configure the standard unit tests for the custom chat model:
|
||||
|
||||
```python
|
||||
# title="tests/unit_tests/test_chat_models.py"
|
||||
from typing import Type
|
||||
|
||||
from my_package.chat_models import MyChatModel
|
||||
from langchain_tests.unit_tests import ChatModelUnitTests
|
||||
|
||||
|
||||
class TestChatParrotLinkUnit(ChatModelUnitTests):
|
||||
@property
|
||||
def chat_model_class(self) -> Type[MyChatModel]:
|
||||
return MyChatModel
|
||||
|
||||
@property
|
||||
def chat_model_params(self) -> dict:
|
||||
# These should be parameters used to initialize your integration for testing
|
||||
return {
|
||||
"model": "bird-brain-001",
|
||||
"temperature": 0,
|
||||
"parrot_buffer_length": 50,
|
||||
}
|
||||
```
|
||||
|
||||
And here is the corresponding snippet for integration tests:
|
||||
|
||||
```python
|
||||
# title="tests/integration_tests/test_chat_models.py"
|
||||
from typing import Type
|
||||
|
||||
from my_package.chat_models import MyChatModel
|
||||
from langchain_tests.integration_tests import ChatModelIntegrationTests
|
||||
|
||||
|
||||
class TestChatParrotLinkIntegration(ChatModelIntegrationTests):
|
||||
@property
|
||||
def chat_model_class(self) -> Type[MyChatModel]:
|
||||
return MyChatModel
|
||||
|
||||
@property
|
||||
def chat_model_params(self) -> dict:
|
||||
# These should be parameters used to initialize your integration for testing
|
||||
return {
|
||||
"model": "bird-brain-001",
|
||||
"temperature": 0,
|
||||
"parrot_buffer_length": 50,
|
||||
}
|
||||
```
|
||||
|
||||
These two snippets should be written into `tests/unit_tests/test_chat_models.py` and `tests/integration_tests/test_chat_models.py`, respectively.
|
||||
|
||||
:::note
|
||||
|
||||
LangChain standard tests test a range of behaviors, from the most basic requirements to optional capabilities like multi-modal support. The above implementation will likely need to be updated to specify any tests that should be ignored. See [below](#skipping-tests) for detail.
|
||||
|
||||
:::
|
||||
|
||||
### Run standard tests
|
||||
|
||||
After setting tests up, you would run these with the following commands from your project root:
|
||||
|
||||
```shell
|
||||
# run unit tests without network access
|
||||
pytest --disable-socket --allow-unix-socket --asyncio-mode=auto tests/unit_tests
|
||||
|
||||
# run integration tests
|
||||
pytest --asyncio-mode=auto tests/integration_tests
|
||||
```
|
||||
|
||||
Our objective is for the pytest run to be successful. That is,
|
||||
|
||||
1. If a feature is intended to be supported by the model, it passes;
|
||||
2. If a feature is not intended to be supported by the model, it is skipped.
|
||||
|
||||
### Skipping tests
|
||||
|
||||
LangChain standard tests test a range of behaviors, from the most basic requirements (generating a response to a query) to optional capabilities like multi-modal support, tool-calling, or support for messages generated from other providers. Tests for "optional" capabilities are controlled via a [set of properties](https://python.langchain.com/api_reference/standard_tests/unit_tests/langchain_tests.unit_tests.chat_models.ChatModelTests.html) that can be overridden on the test model subclass.
|
||||
|
||||
|
||||
### Test suite information and troubleshooting
|
||||
|
||||
What tests are run to test this integration?
|
||||
|
||||
If a test fails, what does that mean?
|
||||
|
||||
You can find information on the tests run for this integration in the [Standard Tests API Reference](https://python.langchain.com/api_reference/standard_tests/index.html).
|
||||
|
||||
// TODO: link to exact page for this integration test suite information
|
||||
@@ -4,14 +4,16 @@ pagination_next: contributing/how_to/integrations/package
|
||||
|
||||
# Contribute Integrations
|
||||
|
||||
LangChain integrations are packages that provide access to language models, vector stores, and other components that can be used in LangChain.
|
||||
Integrations are a core component of LangChain.
|
||||
LangChain provides standard interfaces for several different components (language models, vector stores, etc) that are crucial when building LLM applications.
|
||||
|
||||
This guide will walk you through how to contribute new integrations to LangChain, by
|
||||
publishing an integration package to PyPi, and adding documentation for it
|
||||
to the LangChain Monorepo.
|
||||
|
||||
These instructions will evolve over the next few months as we improve our integration
|
||||
processes.
|
||||
## Why contribute an integration to LangChain?
|
||||
|
||||
- **Discoverability:** LangChain is the most used framework for building LLM applications, with over 20 million monthly downloads. LangChain integrations are discoverable by a large community of GenAI builders.
|
||||
- **Interoptability:** LangChain components expose a standard interface, allowing developers to easily swap them for each other. If you implement a LangChain integration, any developer using a different component will easily be able to swap yours in.
|
||||
- **Best Practices:** Through their standard interface, LangChain components encourage and facilitate best practices (streaming, async, etc)
|
||||
|
||||
|
||||
## Components to Integrate
|
||||
|
||||
@@ -22,8 +24,7 @@ supported in LangChain
|
||||
|
||||
:::
|
||||
|
||||
While any component can be integrated into LangChain, at this time we are only accepting
|
||||
new integrations in the docs of the following kinds:
|
||||
While any component can be integrated into LangChain, there are specific types of integrations we encourage more:
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
@@ -60,18 +61,53 @@ new integrations in the docs of the following kinds:
|
||||
|
||||
## How to contribute an integration
|
||||
|
||||
The only step necessary to "be" a LangChain integration is to add documentation
|
||||
that will render on this site (https://python.langchain.com/).
|
||||
In order to contribute an integration, you should follow these steps:
|
||||
|
||||
As a prerequisite to adding your integration to our documentation, you must:
|
||||
1. Confirm that your integration is in the [list of components](#components-to-integrate) we are currently encouraging.
|
||||
2. [Bootstrap your integration](/docs/contributing/how_to/integrations/package/).
|
||||
3. Implement and test your integration following the [component-specific guides](#component-specific-guides).
|
||||
4. [Publish your integration](/docs/contributing/how_to/integrations/publish/) in a Python package to PyPi.
|
||||
5. [Optional] Open and merge a PR to add documentation for your integration to the official LangChain docs.
|
||||
6. [Optional] Engage with the LangChain team for joint co-marketing ([see below](#co-marketing)).
|
||||
|
||||
1. Confirm that your integration is in the [list of components](#components-to-integrate) we are currently accepting.
|
||||
2. [Implement your package](./package.mdx) and publish it to a public github repository.
|
||||
3. [Implement the standard tests](./standard_tests) for your integration and successfully run them.
|
||||
4. [Publish your integration](./publish.mdx) by publishing the package to PyPi and add docs in the `docs/docs/integrations` directory of the LangChain monorepo.
|
||||
## Component-specific guides
|
||||
|
||||
Once you have completed these steps, you can submit a PR to the LangChain monorepo to add your integration to the documentation.
|
||||
The guides below walk you through both implementing and testing specific components:
|
||||
|
||||
## Further Reading
|
||||
- [Chat Models](/docs/contributing/how_to/integrations/chat_models)
|
||||
- [Tools]
|
||||
- [Toolkits]
|
||||
- [Retrievers]
|
||||
- [Document Loaders]
|
||||
- [Vector Stores]
|
||||
- [Embedding Models]
|
||||
|
||||
To get started, let's learn [how to bootstrap a new integration package](./package.mdx) for LangChain.
|
||||
## Standard Tests
|
||||
|
||||
Testing is a critical part of the development process that ensures your code works as expected and meets the desired quality standards.
|
||||
In the LangChain ecosystem, we have 2 main types of tests: **unit tests** and **integration tests**.
|
||||
These standard tests help maintain compatibility between different components and ensure reliability.
|
||||
|
||||
**Unit Tests**: Unit tests are designed to validate the smallest parts of your code—individual functions or methods—ensuring they work as expected in isolation. They do not rely on external systems or integrations.
|
||||
|
||||
**Integration Tests**: Integration tests validate that multiple components or systems work together as expected. For tools or integrations relying on external services, these tests often ensure end-to-end functionality.
|
||||
|
||||
Each type of integration has its own set of standard tests. Please see the relevant [component-specific guide](#component-specific-guides) for details on testing your integration.
|
||||
|
||||
## Co-Marketing
|
||||
|
||||
With over 20 million monthly downloads, LangChain has a large audience of developers building LLM applications.
|
||||
Besides just adding integrations, we also like to show them examples of cool tools or APIs they can use.
|
||||
|
||||
While traditionally called "co-marketing", we like to thing of this more as "co-education".
|
||||
For that reason, while we are happy to highlight your integration through our social media channels, we prefer to highlight examples that also serve some educational purpose.
|
||||
Our main social media channels are Twitter and LinkedIn.
|
||||
|
||||
Here are some heuristics for types of content we are excited to promote:
|
||||
|
||||
- **Integration announcement:** If you announce the integration with a link to the LangChain documentation page, we are happy to re-tweet/re-share on Twitter/LinkedIn.
|
||||
- **Educational content:** We highlight good educational content on the weekends - if you write a good blog or make a good YouTube video, we are happy to share there! Note that we prefer content that is NOT framed as "here's how to use integration XYZ", but rather "here's how to do ABC", as we find that is more educational and helpful for developers.
|
||||
- **End-to-end applications:** End-to-end applications are great resources for developers looking to build. We prefer to highlight applications that are more complex/agentic in nature, and that use [LangGraph](https://github.com/langchain-ai/langgraph) as the orchestration framework. We get particularly excited about anything involving long-term memory, human-in-the-loop interaction patterns, or multi-agent architectures.
|
||||
- **Research:** We love highlighting novel research! Whether it is research built on top of LangChain or that integrates with it.
|
||||
|
||||
// TODO: set up some form to gather these requests.
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
pagination_next: contributing/how_to/integrations/standard_tests
|
||||
pagination_prev: contributing/how_to/integrations/index
|
||||
pagination_next: contributing/how_to/integrations/index
|
||||
pagination_prev: null
|
||||
---
|
||||
# How to bootstrap a new integration package
|
||||
|
||||
@@ -46,7 +46,7 @@ We will also add some `test` dependencies in a separate poetry dependency group.
|
||||
you are not using Poetry, we recommend adding these in a way that won't package them
|
||||
with your published package, or just installing them separately when you run tests.
|
||||
|
||||
`langchain-tests` will provide the [standard tests](../standard_tests) we will use later.
|
||||
`langchain-tests` will provide the [standard tests](/docs/contributing/how_to/integrations/#standard-tests) we will use later.
|
||||
We recommended pinning these to the latest version: <img src="https://img.shields.io/pypi/v/langchain-tests" style={{position:"relative",top:4,left:3}} />
|
||||
|
||||
Note: Replace `<latest_version>` with the latest version of `langchain-tests` below.
|
||||
@@ -64,203 +64,7 @@ poetry install --with test
|
||||
|
||||
You're now ready to start writing your integration package!
|
||||
|
||||
## Writing your integration
|
||||
|
||||
Let's say you're building a simple integration package that provides a `ChatParrotLink`
|
||||
chat model integration for LangChain. Here's a simple example of what your project
|
||||
structure might look like:
|
||||
|
||||
```plaintext
|
||||
langchain-parrot-link/
|
||||
├── langchain_parrot_link/
|
||||
│ ├── __init__.py
|
||||
│ └── chat_models.py
|
||||
├── tests/
|
||||
│ ├── __init__.py
|
||||
│ └── test_chat_models.py
|
||||
├── pyproject.toml
|
||||
└── README.md
|
||||
```
|
||||
|
||||
All of these files should already exist from step 1, except for
|
||||
`chat_models.py` and `test_chat_models.py`! We will implement `test_chat_models.py`
|
||||
later, following the [standard tests](../standard_tests) guide.
|
||||
|
||||
To implement `chat_models.py`, let's copy the implementation from our
|
||||
[Custom Chat Model Guide](../../../../how_to/custom_chat_model).
|
||||
|
||||
<details>
|
||||
<summary>chat_models.py</summary>
|
||||
```python title="langchain_parrot_link/chat_models.py"
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
|
||||
from langchain_core.callbacks import (
|
||||
CallbackManagerForLLMRun,
|
||||
)
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.messages import (
|
||||
AIMessage,
|
||||
AIMessageChunk,
|
||||
BaseMessage,
|
||||
)
|
||||
from langchain_core.messages.ai import UsageMetadata
|
||||
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
|
||||
from pydantic import Field
|
||||
|
||||
|
||||
class ChatParrotLink(BaseChatModel):
|
||||
"""A custom chat model that echoes the first `parrot_buffer_length` characters
|
||||
of the input.
|
||||
|
||||
When contributing an implementation to LangChain, carefully document
|
||||
the model including the initialization parameters, include
|
||||
an example of how to initialize the model and include any relevant
|
||||
links to the underlying models documentation or API.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model = ChatParrotLink(parrot_buffer_length=2, model="bird-brain-001")
|
||||
result = model.invoke([HumanMessage(content="hello")])
|
||||
result = model.batch([[HumanMessage(content="hello")],
|
||||
[HumanMessage(content="world")]])
|
||||
"""
|
||||
|
||||
model_name: str = Field(alias="model")
|
||||
"""The name of the model"""
|
||||
parrot_buffer_length: int
|
||||
"""The number of characters from the last message of the prompt to be echoed."""
|
||||
temperature: Optional[float] = None
|
||||
max_tokens: Optional[int] = None
|
||||
timeout: Optional[int] = None
|
||||
stop: Optional[List[str]] = None
|
||||
max_retries: int = 2
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
messages: List[BaseMessage],
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any,
|
||||
) -> ChatResult:
|
||||
"""Override the _generate method to implement the chat model logic.
|
||||
|
||||
This can be a call to an API, a call to a local model, or any other
|
||||
implementation that generates a response to the input prompt.
|
||||
|
||||
Args:
|
||||
messages: the prompt composed of a list of messages.
|
||||
stop: a list of strings on which the model should stop generating.
|
||||
If generation stops due to a stop token, the stop token itself
|
||||
SHOULD BE INCLUDED as part of the output. This is not enforced
|
||||
across models right now, but it's a good practice to follow since
|
||||
it makes it much easier to parse the output of the model
|
||||
downstream and understand why generation stopped.
|
||||
run_manager: A run manager with callbacks for the LLM.
|
||||
"""
|
||||
# Replace this with actual logic to generate a response from a list
|
||||
# of messages.
|
||||
last_message = messages[-1]
|
||||
tokens = last_message.content[: self.parrot_buffer_length]
|
||||
ct_input_tokens = sum(len(message.content) for message in messages)
|
||||
ct_output_tokens = len(tokens)
|
||||
message = AIMessage(
|
||||
content=tokens,
|
||||
additional_kwargs={}, # Used to add additional payload to the message
|
||||
response_metadata={ # Use for response metadata
|
||||
"time_in_seconds": 3,
|
||||
},
|
||||
usage_metadata={
|
||||
"input_tokens": ct_input_tokens,
|
||||
"output_tokens": ct_output_tokens,
|
||||
"total_tokens": ct_input_tokens + ct_output_tokens,
|
||||
},
|
||||
)
|
||||
##
|
||||
|
||||
generation = ChatGeneration(message=message)
|
||||
return ChatResult(generations=[generation])
|
||||
|
||||
def _stream(
|
||||
self,
|
||||
messages: List[BaseMessage],
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[CallbackManagerForLLMRun] = None,
|
||||
**kwargs: Any,
|
||||
) -> Iterator[ChatGenerationChunk]:
|
||||
"""Stream the output of the model.
|
||||
|
||||
This method should be implemented if the model can generate output
|
||||
in a streaming fashion. If the model does not support streaming,
|
||||
do not implement it. In that case streaming requests will be automatically
|
||||
handled by the _generate method.
|
||||
|
||||
Args:
|
||||
messages: the prompt composed of a list of messages.
|
||||
stop: a list of strings on which the model should stop generating.
|
||||
If generation stops due to a stop token, the stop token itself
|
||||
SHOULD BE INCLUDED as part of the output. This is not enforced
|
||||
across models right now, but it's a good practice to follow since
|
||||
it makes it much easier to parse the output of the model
|
||||
downstream and understand why generation stopped.
|
||||
run_manager: A run manager with callbacks for the LLM.
|
||||
"""
|
||||
last_message = messages[-1]
|
||||
tokens = str(last_message.content[: self.parrot_buffer_length])
|
||||
ct_input_tokens = sum(len(message.content) for message in messages)
|
||||
|
||||
for token in tokens:
|
||||
usage_metadata = UsageMetadata(
|
||||
{
|
||||
"input_tokens": ct_input_tokens,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": ct_input_tokens + 1,
|
||||
}
|
||||
)
|
||||
ct_input_tokens = 0
|
||||
chunk = ChatGenerationChunk(
|
||||
message=AIMessageChunk(content=token, usage_metadata=usage_metadata)
|
||||
)
|
||||
|
||||
if run_manager:
|
||||
# This is optional in newer versions of LangChain
|
||||
# The on_llm_new_token will be called automatically
|
||||
run_manager.on_llm_new_token(token, chunk=chunk)
|
||||
|
||||
yield chunk
|
||||
|
||||
# Let's add some other information (e.g., response metadata)
|
||||
chunk = ChatGenerationChunk(
|
||||
message=AIMessageChunk(content="", response_metadata={"time_in_sec": 3})
|
||||
)
|
||||
if run_manager:
|
||||
# This is optional in newer versions of LangChain
|
||||
# The on_llm_new_token will be called automatically
|
||||
run_manager.on_llm_new_token(token, chunk=chunk)
|
||||
yield chunk
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
"""Get the type of language model used by this chat model."""
|
||||
return "echoing-chat-model-advanced"
|
||||
|
||||
@property
|
||||
def _identifying_params(self) -> Dict[str, Any]:
|
||||
"""Return a dictionary of identifying parameters.
|
||||
|
||||
This information is used by the LangChain callback system, which
|
||||
is used for tracing purposes make it possible to monitor LLMs.
|
||||
"""
|
||||
return {
|
||||
# The model name allows users to specify custom token counting
|
||||
# rules in LLM monitoring applications (e.g., in LangSmith users
|
||||
# can provide per token pricing for their model and monitor
|
||||
# costs for the given LLM.)
|
||||
"model_name": self.model_name,
|
||||
}
|
||||
```
|
||||
</details>
|
||||
See our [component-specific guides](/docs/contributing/how_to/integrations/#component-specific-guides) for detail on implementing and testing each component.
|
||||
|
||||
## Push your package to a public Github repository
|
||||
|
||||
@@ -272,4 +76,4 @@ This is only required if you want to publish your integration in the LangChain d
|
||||
|
||||
## Next Steps
|
||||
|
||||
Now that you've implemented your package, you can move on to [testing your integration](../standard_tests) for your integration and successfully run them.
|
||||
Now that you've bootstrapped your package, you can move on to [implementing and testing](/docs/contributing/how_to/integrations/#component-specific-guides) your integration.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
pagination_prev: contributing/how_to/integrations/standard_tests
|
||||
pagination_prev: contributing/how_to/integrations/index
|
||||
pagination_next: null
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@ Now that your package is implemented and tested, you can:
|
||||
|
||||
## Publishing your package to PyPi
|
||||
|
||||
This guide assumes you have already implemented your package and written tests for it. If you haven't done that yet, please refer to the [implementation guide](../package) and the [testing guide](../standard_tests).
|
||||
This guide assumes you have already implemented your package and written tests for it. If you haven't done that yet, please refer to the [component-specific guides](/docs/contributing/how_to/integrations/#component-specific-guides).
|
||||
|
||||
Note that Poetry is not required to publish a package to PyPi, and we're using it in this guide end-to-end for convenience.
|
||||
You are welcome to publish your package using any other method you prefer.
|
||||
|
||||
@@ -1,497 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "raw",
|
||||
"metadata": {
|
||||
"vscode": {
|
||||
"languageId": "raw"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"---\n",
|
||||
"pagination_next: contributing/how_to/integrations/publish\n",
|
||||
"pagination_prev: contributing/how_to/integrations/package\n",
|
||||
"---"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# How to add standard tests to an integration\n",
|
||||
"\n",
|
||||
"When creating either a custom class for yourself or to publish in a LangChain integration, it is important to add standard tests to ensure it works as expected. This guide will show you how to add standard tests to a custom chat model, and you can **[Skip to the test templates](#standard-test-templates-per-component)** for implementing tests for each integration type.\n",
|
||||
"\n",
|
||||
"## Setup\n",
|
||||
"\n",
|
||||
"If you're coming from the [previous guide](../package), you have already installed these dependencies, and you can skip this section.\n",
|
||||
"\n",
|
||||
"First, let's install 2 dependencies:\n",
|
||||
"\n",
|
||||
"- `langchain-core` will define the interfaces we want to import to define our custom tool.\n",
|
||||
"- `langchain-tests` will provide the standard tests we want to use. Recommended to pin to the latest version: <img src=\"https://img.shields.io/pypi/v/langchain-tests\" style={{position:\"relative\",top:4,left:3}} />\n",
|
||||
"\n",
|
||||
":::note\n",
|
||||
"\n",
|
||||
"Because added tests in new versions of `langchain-tests` can break your CI/CD pipelines, we recommend pinning the \n",
|
||||
"version of `langchain-tests` to avoid unexpected changes.\n",
|
||||
"\n",
|
||||
":::\n",
|
||||
"\n",
|
||||
"import Tabs from '@theme/Tabs';\n",
|
||||
"import TabItem from '@theme/TabItem';\n",
|
||||
"\n",
|
||||
"<Tabs>\n",
|
||||
" <TabItem value=\"poetry\" label=\"Poetry\" default>\n",
|
||||
"If you followed the [previous guide](../package), you should already have these dependencies installed!\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"poetry add langchain-core\n",
|
||||
"poetry add --group test pytest pytest-socket pytest-asyncio langchain-tests==<latest_version>\n",
|
||||
"poetry install --with test\n",
|
||||
"```\n",
|
||||
" </TabItem>\n",
|
||||
" <TabItem value=\"pip\" label=\"Pip\">\n",
|
||||
"```bash\n",
|
||||
"pip install -U langchain-core pytest pytest-socket pytest-asyncio langchain-tests\n",
|
||||
"\n",
|
||||
"# install current package in editable mode\n",
|
||||
"pip install --editable .\n",
|
||||
"```\n",
|
||||
" </TabItem>\n",
|
||||
"</Tabs>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let's say we're publishing a package, `langchain_parrot_link`, that exposes the chat model from the [guide on implementing the package](../package). We can add the standard tests to the package by following the steps below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"And we'll assume you've structured your package the same way as the main LangChain\n",
|
||||
"packages:\n",
|
||||
"\n",
|
||||
"```plaintext\n",
|
||||
"langchain-parrot-link/\n",
|
||||
"├── langchain_parrot_link/\n",
|
||||
"│ ├── __init__.py\n",
|
||||
"│ └── chat_models.py\n",
|
||||
"├── tests/\n",
|
||||
"│ ├── __init__.py\n",
|
||||
"│ └── test_chat_models.py\n",
|
||||
"├── pyproject.toml\n",
|
||||
"└── README.md\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Add and configure standard tests\n",
|
||||
"\n",
|
||||
"There are 2 namespaces in the `langchain-tests` package: \n",
|
||||
"\n",
|
||||
"- [unit tests](../../../concepts/testing.mdx#unit-tests) (`langchain_tests.unit_tests`): designed to be used to test the component in isolation and without access to external services\n",
|
||||
"- [integration tests](../../../concepts/testing.mdx#unit-tests) (`langchain_tests.integration_tests`): designed to be used to test the component with access to external services (in particular, the external service that the component is designed to interact with).\n",
|
||||
"\n",
|
||||
"Both types of tests are implemented as [`pytest` class-based test suites](https://docs.pytest.org/en/7.1.x/getting-started.html#group-multiple-tests-in-a-class).\n",
|
||||
"\n",
|
||||
"By subclassing the base classes for each type of standard test (see below), you get all of the standard tests for that type, and you\n",
|
||||
"can override the properties that the test suite uses to configure the tests.\n",
|
||||
"\n",
|
||||
"### Standard chat model tests\n",
|
||||
"\n",
|
||||
"Here's how you would configure the standard unit tests for the custom chat model:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/unit_tests/test_chat_models.py\"\n",
|
||||
"from typing import Tuple, Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.chat_models import ChatParrotLink\n",
|
||||
"from langchain_tests.unit_tests import ChatModelUnitTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestChatParrotLinkUnit(ChatModelUnitTests):\n",
|
||||
" @property\n",
|
||||
" def chat_model_class(self) -> Type[ChatParrotLink]:\n",
|
||||
" return ChatParrotLink\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def chat_model_params(self) -> dict:\n",
|
||||
" return {\n",
|
||||
" \"model\": \"bird-brain-001\",\n",
|
||||
" \"temperature\": 0,\n",
|
||||
" \"parrot_buffer_length\": 50,\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/integration_tests/test_chat_models.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.chat_models import ChatParrotLink\n",
|
||||
"from langchain_tests.integration_tests import ChatModelIntegrationTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestChatParrotLinkIntegration(ChatModelIntegrationTests):\n",
|
||||
" @property\n",
|
||||
" def chat_model_class(self) -> Type[ChatParrotLink]:\n",
|
||||
" return ChatParrotLink\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def chat_model_params(self) -> dict:\n",
|
||||
" return {\n",
|
||||
" \"model\": \"bird-brain-001\",\n",
|
||||
" \"temperature\": 0,\n",
|
||||
" \"parrot_buffer_length\": 50,\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"and you would run these with the following commands from your project root\n",
|
||||
"\n",
|
||||
"<Tabs>\n",
|
||||
" <TabItem value=\"poetry\" label=\"Poetry\" default>\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"# run unit tests without network access\n",
|
||||
"poetry run pytest --disable-socket --allow-unix-socket --asyncio-mode=auto tests/unit_tests\n",
|
||||
"\n",
|
||||
"# run integration tests\n",
|
||||
"poetry run pytest --asyncio-mode=auto tests/integration_tests\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
" </TabItem>\n",
|
||||
" <TabItem value=\"pip\" label=\"Pip\">\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"# run unit tests without network access\n",
|
||||
"pytest --disable-socket --allow-unix-socket --asyncio-mode=auto tests/unit_tests\n",
|
||||
"\n",
|
||||
"# run integration tests\n",
|
||||
"pytest --asyncio-mode=auto tests/integration_tests\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
" </TabItem>\n",
|
||||
"</Tabs>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Test suite information and troubleshooting\n",
|
||||
"\n",
|
||||
"For a full list of the standard test suites that are available, as well as\n",
|
||||
"information on which tests are included and how to troubleshoot common issues,\n",
|
||||
"see the [Standard Tests API Reference](https://python.langchain.com/api_reference/standard_tests/index.html).\n",
|
||||
"\n",
|
||||
"An increasing number of troubleshooting guides are being added to this documentation,\n",
|
||||
"and if you're interested in contributing, feel free to add docstrings to tests in \n",
|
||||
"[Github](https://github.com/langchain-ai/langchain/tree/master/libs/standard-tests/langchain_tests)!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Standard test templates per component:\n",
|
||||
"\n",
|
||||
"Above, we implement the **unit** and **integration** standard tests for a tool. Below are the templates for implementing the standard tests for each component:\n",
|
||||
"\n",
|
||||
"<details>\n",
|
||||
" <summary>Chat Models</summary>\n",
|
||||
" <p>Note: The standard tests for chat models are implemented in the example in the main body of this guide too.</p>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/unit_tests/test_chat_models.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.chat_models import ChatParrotLink\n",
|
||||
"from langchain_tests.unit_tests import ChatModelUnitTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestChatParrotLinkUnit(ChatModelUnitTests):\n",
|
||||
" @property\n",
|
||||
" def chat_model_class(self) -> Type[ChatParrotLink]:\n",
|
||||
" return ChatParrotLink\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def chat_model_params(self) -> dict:\n",
|
||||
" return {\n",
|
||||
" \"model\": \"bird-brain-001\",\n",
|
||||
" \"temperature\": 0,\n",
|
||||
" \"parrot_buffer_length\": 50,\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/integration_tests/test_chat_models.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.chat_models import ChatParrotLink\n",
|
||||
"from langchain_tests.integration_tests import ChatModelIntegrationTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestChatParrotLinkIntegration(ChatModelIntegrationTests):\n",
|
||||
" @property\n",
|
||||
" def chat_model_class(self) -> Type[ChatParrotLink]:\n",
|
||||
" return ChatParrotLink\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def chat_model_params(self) -> dict:\n",
|
||||
" return {\n",
|
||||
" \"model\": \"bird-brain-001\",\n",
|
||||
" \"temperature\": 0,\n",
|
||||
" \"parrot_buffer_length\": 50,\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"</details>\n",
|
||||
"<details>\n",
|
||||
" <summary>Embedding Models</summary>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/unit_tests/test_embeddings.py\"\n",
|
||||
"from typing import Tuple, Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.embeddings import ParrotLinkEmbeddings\n",
|
||||
"from langchain_tests.unit_tests import EmbeddingsUnitTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestParrotLinkEmbeddingsUnit(EmbeddingsUnitTests):\n",
|
||||
" @property\n",
|
||||
" def embeddings_class(self) -> Type[ParrotLinkEmbeddings]:\n",
|
||||
" return ParrotLinkEmbeddings\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def embedding_model_params(self) -> dict:\n",
|
||||
" return {\"model\": \"nest-embed-001\", \"temperature\": 0}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/integration_tests/test_embeddings.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.embeddings import ParrotLinkEmbeddings\n",
|
||||
"from langchain_tests.integration_tests import EmbeddingsIntegrationTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestParrotLinkEmbeddingsIntegration(EmbeddingsIntegrationTests):\n",
|
||||
" @property\n",
|
||||
" def embeddings_class(self) -> Type[ParrotLinkEmbeddings]:\n",
|
||||
" return ParrotLinkEmbeddings\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def embedding_model_params(self) -> dict:\n",
|
||||
" return {\"model\": \"nest-embed-001\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"</details>\n",
|
||||
"<details>\n",
|
||||
" <summary>Tools/Toolkits</summary>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/unit_tests/test_tools.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.tools import ParrotMultiplyTool\n",
|
||||
"from langchain_tests.unit_tests import ToolsUnitTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestParrotMultiplyToolUnit(ToolsUnitTests):\n",
|
||||
" @property\n",
|
||||
" def tool_constructor(self) -> Type[ParrotMultiplyTool]:\n",
|
||||
" return ParrotMultiplyTool\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def tool_constructor_params(self) -> dict:\n",
|
||||
" # if your tool constructor instead required initialization arguments like\n",
|
||||
" # `def __init__(self, some_arg: int):`, you would return those here\n",
|
||||
" # as a dictionary, e.g.: `return {'some_arg': 42}`\n",
|
||||
" return {}\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def tool_invoke_params_example(self) -> dict:\n",
|
||||
" \"\"\"\n",
|
||||
" Returns a dictionary representing the \"args\" of an example tool call.\n",
|
||||
"\n",
|
||||
" This should NOT be a ToolCall dict - i.e. it should not\n",
|
||||
" have {\"name\", \"id\", \"args\"} keys.\n",
|
||||
" \"\"\"\n",
|
||||
" return {\"a\": 2, \"b\": 3}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/integration_tests/test_tools.py\"\n",
|
||||
"from typing import Type\n",
|
||||
"\n",
|
||||
"from langchain_parrot_link.tools import ParrotMultiplyTool\n",
|
||||
"from langchain_tests.integration_tests import ToolsIntegrationTests\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestParrotMultiplyToolIntegration(ToolsIntegrationTests):\n",
|
||||
" @property\n",
|
||||
" def tool_constructor(self) -> Type[ParrotMultiplyTool]:\n",
|
||||
" return ParrotMultiplyTool\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def tool_constructor_params(self) -> dict:\n",
|
||||
" # if your tool constructor instead required initialization arguments like\n",
|
||||
" # `def __init__(self, some_arg: int):`, you would return those here\n",
|
||||
" # as a dictionary, e.g.: `return {'some_arg': 42}`\n",
|
||||
" return {}\n",
|
||||
"\n",
|
||||
" @property\n",
|
||||
" def tool_invoke_params_example(self) -> dict:\n",
|
||||
" \"\"\"\n",
|
||||
" Returns a dictionary representing the \"args\" of an example tool call.\n",
|
||||
"\n",
|
||||
" This should NOT be a ToolCall dict - i.e. it should not\n",
|
||||
" have {\"name\", \"id\", \"args\"} keys.\n",
|
||||
" \"\"\"\n",
|
||||
" return {\"a\": 2, \"b\": 3}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"</details>\n",
|
||||
"<details>\n",
|
||||
" <summary>Vector Stores</summary>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# title=\"tests/integration_tests/test_vectorstores_sync.py\"\n",
|
||||
"\n",
|
||||
"from typing import AsyncGenerator, Generator\n",
|
||||
"\n",
|
||||
"import pytest\n",
|
||||
"from langchain_core.vectorstores import VectorStore\n",
|
||||
"from langchain_parrot_link.vectorstores import ParrotVectorStore\n",
|
||||
"from langchain_standard_tests.integration_tests.vectorstores import (\n",
|
||||
" AsyncReadWriteTestSuite,\n",
|
||||
" ReadWriteTestSuite,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestSync(ReadWriteTestSuite):\n",
|
||||
" @pytest.fixture()\n",
|
||||
" def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore\n",
|
||||
" \"\"\"Get an empty vectorstore for unit tests.\"\"\"\n",
|
||||
" store = ParrotVectorStore()\n",
|
||||
" # note: store should be EMPTY at this point\n",
|
||||
" # if you need to delete data, you may do so here\n",
|
||||
" try:\n",
|
||||
" yield store\n",
|
||||
" finally:\n",
|
||||
" # cleanup operations, or deleting data\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class TestAsync(AsyncReadWriteTestSuite):\n",
|
||||
" @pytest.fixture()\n",
|
||||
" async def vectorstore(self) -> AsyncGenerator[VectorStore, None]: # type: ignore\n",
|
||||
" \"\"\"Get an empty vectorstore for unit tests.\"\"\"\n",
|
||||
" store = ParrotVectorStore()\n",
|
||||
" # note: store should be EMPTY at this point\n",
|
||||
" # if you need to delete data, you may do so here\n",
|
||||
" try:\n",
|
||||
" yield store\n",
|
||||
" finally:\n",
|
||||
" # cleanup operations, or deleting data\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"</details>"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -17,7 +17,6 @@ More coming soon! We are working on tutorials to help you make your first contri
|
||||
- [**Documentation**](how_to/documentation/index.mdx): Help improve our docs, including this one!
|
||||
- [**Code**](how_to/code/index.mdx): Help us write code, fix bugs, or improve our infrastructure.
|
||||
- [**Integrations**](how_to/integrations/index.mdx): Help us integrate with your favorite vendors and tools.
|
||||
- [**Standard Tests**](how_to/integrations/standard_tests): Ensure your integration passes an expected set of tests.
|
||||
|
||||
## Reference
|
||||
|
||||
|
||||
@@ -113,6 +113,10 @@
|
||||
{
|
||||
"source": "/docs/contributing/:path((?:faq|repo_structure|review_process)/?)",
|
||||
"destination": "/docs/contributing/reference/:path"
|
||||
},
|
||||
{
|
||||
"source": "/docs/contributing/how_to/integrations/standard_tests(/?)",
|
||||
"destination": "/docs/contributing/how_to/integrations/#standard-tests"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -79,10 +79,22 @@ def my_adder(a: int, b: int) -> int:
|
||||
class ChatModelTests(BaseStandardTests):
|
||||
@property
|
||||
@abstractmethod
|
||||
def chat_model_class(self) -> Type[BaseChatModel]: ...
|
||||
def chat_model_class(self) -> Type[BaseChatModel]:
|
||||
"""The chat model class to test, e.g., `ChatParrotLink`."""
|
||||
...
|
||||
|
||||
@property
|
||||
def chat_model_params(self) -> dict:
|
||||
"""Initialization parameters for the chat model.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def standard_chat_model_params(self) -> dict:
|
||||
return {"model": "bird-brain-001", "temperature": 0}
|
||||
"""
|
||||
return {}
|
||||
|
||||
@property
|
||||
@@ -103,15 +115,53 @@ class ChatModelTests(BaseStandardTests):
|
||||
|
||||
@property
|
||||
def has_tool_calling(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports tool calling.
|
||||
|
||||
By default, this is determined by whether the chat model's `bind_tools` method
|
||||
is overridden. It typically does not need to be overridden on the test class.
|
||||
"""
|
||||
return self.chat_model_class.bind_tools is not BaseChatModel.bind_tools
|
||||
|
||||
@property
|
||||
def tool_choice_value(self) -> Optional[str]:
|
||||
"""Value to use for tool choice when used in tests."""
|
||||
"""Value to use for tool choice when used in tests.
|
||||
|
||||
Some tests for tool calling features attempt to force tool calling via a
|
||||
`tool_choice` parameter. A common value for this parameter is "any". Defaults
|
||||
to `None`.
|
||||
|
||||
Note: if the value is set to "tool_name", the name of the tool used in each
|
||||
test will be set as the value for `tool_choice`.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def tool_choice_value(self) -> Optional[str]:
|
||||
return "any"
|
||||
"""
|
||||
return None
|
||||
|
||||
@property
|
||||
def has_structured_output(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports structured
|
||||
output.
|
||||
|
||||
By default, this is determined by whether the chat model's
|
||||
`with_structured_output` method is overridden. If the base implementation is
|
||||
intended to be used, this method should be overridden.
|
||||
|
||||
See: https://python.langchain.com/docs/concepts/structured_outputs/
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def has_structured_output(self) -> bool:
|
||||
return True
|
||||
"""
|
||||
return (
|
||||
self.chat_model_class.with_structured_output
|
||||
is not BaseChatModel.with_structured_output
|
||||
@@ -119,22 +169,118 @@ class ChatModelTests(BaseStandardTests):
|
||||
|
||||
@property
|
||||
def supports_image_inputs(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports image inputs.
|
||||
Defaults to ``False``.
|
||||
|
||||
If set to ``True``, the chat model will be tested using content blocks of the form
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
[
|
||||
{"type": "text", "text": "describe the weather in this image"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
|
||||
},
|
||||
]
|
||||
|
||||
See https://python.langchain.com/docs/concepts/multimodality/
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def supports_image_inputs(self) -> bool:
|
||||
return True
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def supports_video_inputs(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports image inputs.
|
||||
Defaults to ``False``. No current tests are written for this feature."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def returns_usage_metadata(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model returns usage metadata
|
||||
on invoke and streaming responses.
|
||||
|
||||
``usage_metadata`` is an optional dict attribute on AIMessages that track input
|
||||
and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def returns_usage_metadata(self) -> bool:
|
||||
return False
|
||||
""" # noqa: E501
|
||||
return True
|
||||
|
||||
@property
|
||||
def supports_anthropic_inputs(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports Anthropic-style
|
||||
inputs.
|
||||
|
||||
These inputs might feature "tool use" and "tool result" content blocks, e.g.,
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
[
|
||||
{"type": "text", "text": "Hmm let me think about that"},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"input": {"fav_color": "green"},
|
||||
"id": "foo",
|
||||
"name": "color_picker",
|
||||
},
|
||||
]
|
||||
|
||||
If set to ``True``, the chat model will be tested using content blocks of this
|
||||
form.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def supports_anthropic_inputs(self) -> bool:
|
||||
return False
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def supports_image_tool_message(self) -> bool:
|
||||
"""Boolean property indicating whether the chat model supports ToolMessages
|
||||
that include image content, e.g.,
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
ToolMessage(
|
||||
content=[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
|
||||
},
|
||||
],
|
||||
tool_call_id="1",
|
||||
name="random_image",
|
||||
)
|
||||
|
||||
If set to ``True``, the chat model will be tested with message sequences that
|
||||
include ToolMessages of this form.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def supports_image_tool_message(self) -> bool:
|
||||
return False
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
@@ -152,6 +298,18 @@ class ChatModelTests(BaseStandardTests):
|
||||
]
|
||||
],
|
||||
]:
|
||||
"""Property controlling what usage metadata details are emitted in both invoke
|
||||
and stream.
|
||||
|
||||
``usage_metadata`` is an optional dict attribute on AIMessages that track input
|
||||
and output tokens: https://python.langchain.com/api_reference/core/messages/langchain_core.messages.ai.UsageMetadata.html
|
||||
|
||||
It includes optional keys ``input_token_details`` and ``output_token_details``
|
||||
that can track usage details associated with special types of tokens, such as
|
||||
cached, audio, or reasoning.
|
||||
|
||||
Only needs to be overridden if these details are supplied.
|
||||
""" # noqa: E501
|
||||
return {"invoke": [], "stream": []}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user