mirror of
https://github.com/hwchase17/langchain.git
synced 2025-05-11 18:16:12 +00:00
53b0a99f37
## Description This PR adds support for Memcached as a usable LLM model cache by adding the ```MemcachedCache``` implementation relying on the [pymemcache](https://github.com/pinterest/pymemcache) client. Unit test-wise, the new integration is generally covered under existing import testing. All new functionality depends on pymemcache if instantiated and used, so to comply with the other cache implementations the PR also adds optional integration tests for ```MemcachedCache```. Since this is a new integration, documentation is added for Memcached as an integration and as an LLM Cache. ## Issue This PR closes #27275 which was originally raised as a discussion in #27035 ## Dependencies There are no new required dependencies for langchain, but [pymemcache](https://github.com/pinterest/pymemcache) is required to instantiate the new ```MemcachedCache```. ## Example Usage ```python3 from langchain.globals import set_llm_cache from langchain_openai import OpenAI from langchain_community.cache import MemcachedCache from pymemcache.client.base import Client llm = OpenAI(model="gpt-3.5-turbo-instruct", n=2, best_of=2) set_llm_cache(MemcachedCache(Client('localhost'))) # The first time, it is not yet in cache, so it should take longer llm.invoke("Which city is the most crowded city in the USA?") # The second time it is, so it goes faster llm.invoke("Which city is the most crowded city in the USA?") ``` --------- Co-authored-by: Erick Friis <erick@langchain.dev>
62 lines
1.8 KiB
Python
62 lines
1.8 KiB
Python
"""
|
|
Test Memcached llm cache functionality. Requires running instance of Memcached on
|
|
localhost default port (11211) and pymemcache
|
|
"""
|
|
|
|
import pytest
|
|
from langchain.globals import get_llm_cache, set_llm_cache
|
|
from langchain_core.outputs import Generation, LLMResult
|
|
|
|
from langchain_community.cache import MemcachedCache
|
|
from tests.unit_tests.llms.fake_llm import FakeLLM
|
|
|
|
DEFAULT_MEMCACHED_URL = "localhost"
|
|
|
|
|
|
@pytest.mark.requires("pymemcache")
|
|
def test_memcached_cache() -> None:
|
|
"""Test general Memcached caching"""
|
|
from pymemcache import Client
|
|
|
|
set_llm_cache(MemcachedCache(Client(DEFAULT_MEMCACHED_URL)))
|
|
llm = FakeLLM()
|
|
|
|
params = llm.dict()
|
|
params["stop"] = None
|
|
llm_string = str(sorted([(k, v) for k, v in params.items()]))
|
|
get_llm_cache().update("foo", llm_string, [Generation(text="fizz")])
|
|
output = llm.generate(["foo"])
|
|
expected_output = LLMResult(
|
|
generations=[[Generation(text="fizz")]],
|
|
llm_output={},
|
|
)
|
|
assert output == expected_output
|
|
# clear the cache
|
|
get_llm_cache().clear()
|
|
|
|
|
|
@pytest.mark.requires("pymemcache")
|
|
def test_memcached_cache_flush() -> None:
|
|
"""Test flushing Memcached cache"""
|
|
from pymemcache import Client
|
|
|
|
set_llm_cache(MemcachedCache(Client(DEFAULT_MEMCACHED_URL)))
|
|
llm = FakeLLM()
|
|
|
|
params = llm.dict()
|
|
params["stop"] = None
|
|
llm_string = str(sorted([(k, v) for k, v in params.items()]))
|
|
get_llm_cache().update("foo", llm_string, [Generation(text="fizz")])
|
|
output = llm.generate(["foo"])
|
|
expected_output = LLMResult(
|
|
generations=[[Generation(text="fizz")]],
|
|
llm_output={},
|
|
)
|
|
assert output == expected_output
|
|
# clear the cache
|
|
get_llm_cache().clear(delay=0, noreply=False)
|
|
|
|
# After cache has been cleared, the result shouldn't be the same
|
|
output = llm.generate(["foo"])
|
|
assert output != expected_output
|