From add31f46d0c6d4e6702960869317146864bd043d Mon Sep 17 00:00:00 2001 From: seray Date: Tue, 9 Apr 2024 22:34:56 +0200 Subject: [PATCH] community[patch]: OpenLLM Async Client Fixes and Timeout Parameter (#20007) Same changes as this merged [PR](https://github.com/langchain-ai/langchain/pull/17478) (https://github.com/langchain-ai/langchain/pull/17478), but for the async client, as the same issues persist. - Replaced 'responses' attribute of OpenLLM's GenerationOutput schema to 'outputs'. reference: https://github.com/bentoml/OpenLLM/blob/66de54eae7e420a3740ddd77862fd7f7b7d8a222/openllm-core/src/openllm_core/_schemas.py#L135 - Added timeout parameter for the async client. --------- Co-authored-by: Seray Arslan --- libs/community/langchain_community/llms/openllm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/community/langchain_community/llms/openllm.py b/libs/community/langchain_community/llms/openllm.py index fa3b03e1f98..5d43e7ea265 100644 --- a/libs/community/langchain_community/llms/openllm.py +++ b/libs/community/langchain_community/llms/openllm.py @@ -308,10 +308,12 @@ class OpenLLM(LLM): self._identifying_params["model_name"], **copied ) if self._client: - async_client = openllm.client.AsyncHTTPClient(self.server_url) + async_client = openllm.client.AsyncHTTPClient(self.server_url, self.timeout) res = ( - await async_client.generate(prompt, **config.model_dump(flatten=True)) - ).responses[0] + (await async_client.generate(prompt, **config.model_dump(flatten=True))) + .outputs[0] + .text + ) else: assert self._runner is not None (