mirror of
https://github.com/hwchase17/langchain.git
synced 2025-09-16 06:53:16 +00:00
feat: ctransformers support async chain (#6859)
- Description: Adding async method for CTransformers - Issue: I've found impossible without this code to run Websockets inside a FastAPI micro service and a CTransformers model. - Tag maintainer: Not necessary yet, I don't like to mention directly - Twitter handle: @_semoal
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""Test C Transformers wrapper."""
|
||||
import pytest
|
||||
|
||||
from langchain.llms import CTransformers
|
||||
from tests.unit_tests.callbacks.fake_callback_handler import FakeCallbackHandler
|
||||
@@ -19,3 +20,20 @@ def test_ctransformers_call() -> None:
|
||||
assert isinstance(output, str)
|
||||
assert len(output) > 1
|
||||
assert 0 < callback_handler.llm_streams <= config["max_new_tokens"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ctransformers_async_inference() -> None:
|
||||
config = {"max_new_tokens": 5}
|
||||
callback_handler = FakeCallbackHandler()
|
||||
|
||||
llm = CTransformers(
|
||||
model="marella/gpt-2-ggml",
|
||||
config=config,
|
||||
callbacks=[callback_handler],
|
||||
)
|
||||
|
||||
output = await llm._acall(prompt="Say foo:")
|
||||
assert isinstance(output, str)
|
||||
assert len(output) > 1
|
||||
assert 0 < callback_handler.llm_streams <= config["max_new_tokens"]
|
||||
|
Reference in New Issue
Block a user