From a873e0fbfb4f315db1563b6e84860c5617b136a8 Mon Sep 17 00:00:00 2001 From: minpeter Date: Fri, 3 Jan 2025 02:15:59 +0900 Subject: [PATCH] community: update documentation and model IDs for FriendliAI provider (#28984) ### Description - In the example, remove `llama-2-13b-chat`, `mixtral-8x7b-instruct-v0-1`. - Fix llm friendli streaming implementation. - Update examples in documentation and remove duplicates. ### Issue N/A ### Dependencies None ### Twitter handle `@friendliai` --- docs/docs/integrations/chat/friendli.ipynb | 74 +++++----- docs/docs/integrations/llms/friendli.ipynb | 135 +++++++++--------- .../providers/{friendli.md => friendli.mdx} | 16 ++- docs/docs/integrations/providers/friendly.md | 32 ----- .../chat_models/friendli.py | 4 +- .../langchain_community/llms/friendli.py | 23 +-- .../tests/unit_tests/llms/test_friendli.py | 30 ++-- 7 files changed, 150 insertions(+), 164 deletions(-) rename docs/docs/integrations/providers/{friendli.md => friendli.mdx} (56%) delete mode 100644 docs/docs/integrations/providers/friendly.md diff --git a/docs/docs/integrations/chat/friendli.ipynb b/docs/docs/integrations/chat/friendli.ipynb index dab9e0bdd1f..019f62a31bf 100644 --- a/docs/docs/integrations/chat/friendli.ipynb +++ b/docs/docs/integrations/chat/friendli.ipynb @@ -2,10 +2,14 @@ "cells": [ { "cell_type": "raw", - "metadata": {}, + "metadata": { + "vscode": { + "languageId": "raw" + } + }, "source": [ "---\n", - "sidebar_label: Friendli\n", + "sidebar_label: ChatFriendli\n", "---" ] }, @@ -37,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -57,13 +61,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from langchain_community.chat_models.friendli import ChatFriendli\n", "\n", - "chat = ChatFriendli(model=\"llama-2-13b-chat\", max_tokens=100, temperature=0)" + "chat = ChatFriendli(model=\"meta-llama-3.1-8b-instruct\", max_tokens=100, temperature=0)" ] }, { @@ -84,16 +88,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\")" + "AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-d47c1056-54e8-4ea9-ad63-07cf74b834b7-0')" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -111,17 +115,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"),\n", - " AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\")]" + "[AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-36775b84-2a7a-48f0-8c68-df23ffffe4b2-0'),\n", + " AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-b204be41-bc06-4d3a-9f74-e66ab1e60e4f-0')]" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -132,16 +136,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "LLMResult(generations=[[ChatGeneration(text=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\", message=AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"))], [ChatGeneration(text=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\", message=AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"))]], llm_output={}, run=[RunInfo(run_id=UUID('a0c2d733-6971-4ae7-beea-653856f4e57c')), RunInfo(run_id=UUID('f3d35e44-ac9a-459a-9e4b-b8e3a73a91e1'))])" + "LLMResult(generations=[[ChatGeneration(text=\"Why don't eggs tell jokes? They'd crack each other up.\", message=AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-2e4cb949-8c51-40d5-92a0-cd0ac577db83-0'))], [ChatGeneration(text=\"Why don't eggs tell jokes? They'd crack each other up.\", message=AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-afcdd1be-463c-4e50-9731-7a9f5958e396-0'))]], llm_output={}, run=[RunInfo(run_id=UUID('2e4cb949-8c51-40d5-92a0-cd0ac577db83')), RunInfo(run_id=UUID('afcdd1be-463c-4e50-9731-7a9f5958e396'))], type='LLMResult')" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -152,18 +156,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " Knock, knock!\n", - "Who's there?\n", - "Cows go.\n", - "Cows go who?\n", - "MOO!" + "Why don't eggs tell jokes? They'd crack each other up." ] } ], @@ -181,16 +181,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\")" + "AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-ba8062fb-68af-47b8-bd7b-d1e01b914744-0')" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -201,17 +201,17 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"),\n", - " AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\")]" + "[AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-5d2c77ab-2637-45da-8bbe-1b1f18a22369-0'),\n", + " AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-f1338470-8b52-4d6e-9428-a694a08ae484-0')]" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -222,16 +222,16 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "LLMResult(generations=[[ChatGeneration(text=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\", message=AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"))], [ChatGeneration(text=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\", message=AIMessage(content=\" Knock, knock!\\nWho's there?\\nCows go.\\nCows go who?\\nMOO!\"))]], llm_output={}, run=[RunInfo(run_id=UUID('f2255321-2d8e-41cc-adbd-3f4facec7573')), RunInfo(run_id=UUID('fcc297d0-6ca9-48cb-9d86-e6f78cade8ee'))])" + "LLMResult(generations=[[ChatGeneration(text=\"Why don't eggs tell jokes? They'd crack each other up.\", message=AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-d4e44569-39cc-40cc-93fc-de53e599fd51-0'))], [ChatGeneration(text=\"Why don't eggs tell jokes? They'd crack each other up.\", message=AIMessage(content=\"Why don't eggs tell jokes? They'd crack each other up.\", additional_kwargs={}, response_metadata={}, id='run-54647cc2-bee3-4154-ad00-2e547993e6d7-0'))]], llm_output={}, run=[RunInfo(run_id=UUID('d4e44569-39cc-40cc-93fc-de53e599fd51')), RunInfo(run_id=UUID('54647cc2-bee3-4154-ad00-2e547993e6d7'))], type='LLMResult')" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -242,18 +242,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " Knock, knock!\n", - "Who's there?\n", - "Cows go.\n", - "Cows go who?\n", - "MOO!" + "Why don't eggs tell jokes? They'd crack each other up." ] } ], @@ -265,7 +261,7 @@ ], "metadata": { "kernelspec": { - "display_name": "langchain", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -279,7 +275,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/docs/docs/integrations/llms/friendli.ipynb b/docs/docs/integrations/llms/friendli.ipynb index 529b6aec7a0..74978a39aa5 100644 --- a/docs/docs/integrations/llms/friendli.ipynb +++ b/docs/docs/integrations/llms/friendli.ipynb @@ -29,7 +29,7 @@ "Ensure the `langchain_community` and `friendli-client` are installed.\n", "\n", "```sh\n", - "pip install -U langchain-community friendli-client.\n", + "pip install -U langchain-community friendli-client\n", "```\n", "\n", "Sign in to [Friendli Suite](https://suite.friendli.ai/) to create a Personal Access Token, and set it as the `FRIENDLI_TOKEN` environment." @@ -40,13 +40,20 @@ "execution_count": 1, "metadata": {}, "outputs": [], - "source": ["import getpass\nimport os\n\nif \"FRIENDLI_TOKEN\" not in os.environ:\n os.environ[\"FRIENDLI_TOKEN\"] = getpass.getpass(\"Friendi Personal Access Token: \")"] + "source": [ + "import getpass\n", + "import os\n", + "\n", + "if \"FRIENDLI_TOKEN\" not in os.environ:\n", + " os.environ[\"FRIENDLI_TOKEN\"] = getpass.getpass(\"Friendi Personal Access Token: \")" + ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You can initialize a Friendli chat model with selecting the model you want to use. The default model is `mixtral-8x7b-instruct-v0-1`. You can check the available models at [docs.friendli.ai](https://docs.periflow.ai/guides/serverless_endpoints/pricing#text-generation-models)." + "You can initialize a Friendli chat model with selecting the model you want to use. \n", + "The default model is `meta-llama-3.1-8b-instruct`. You can check the available models at [friendli.ai/docs](https://friendli.ai/docs/guides/serverless_endpoints/pricing#text-generation-models)." ] }, { @@ -54,7 +61,11 @@ "execution_count": 2, "metadata": {}, "outputs": [], - "source": ["from langchain_community.llms.friendli import Friendli\n\nllm = Friendli(model=\"mixtral-8x7b-instruct-v0-1\", max_tokens=100, temperature=0)"] + "source": [ + "from langchain_community.llms.friendli import Friendli\n", + "\n", + "llm = Friendli(model=\"meta-llama-3.1-8b-instruct\", max_tokens=100, temperature=0)" + ] }, { "cell_type": "markdown", @@ -80,7 +91,7 @@ { "data": { "text/plain": [ - "'Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"'" + "\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\"" ] }, "execution_count": 3, @@ -88,7 +99,9 @@ "output_type": "execute_result" } ], - "source": ["llm.invoke(\"Tell me a joke.\")"] + "source": [ + "llm.invoke(\"Tell me a joke.\")" + ] }, { "cell_type": "code", @@ -98,8 +111,8 @@ { "data": { "text/plain": [ - "['Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"',\n", - " 'Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"']" + "[\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\",\n", + " \" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\"]" ] }, "execution_count": 4, @@ -107,7 +120,9 @@ "output_type": "execute_result" } ], - "source": ["llm.batch([\"Tell me a joke.\", \"Tell me a joke.\"])"] + "source": [ + "llm.batch([\"Tell me a joke.\", \"Tell me a joke.\"])" + ] }, { "cell_type": "code", @@ -117,7 +132,7 @@ { "data": { "text/plain": [ - "LLMResult(generations=[[Generation(text='Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"')], [Generation(text='Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"')]], llm_output={'model': 'mixtral-8x7b-instruct-v0-1'}, run=[RunInfo(run_id=UUID('a2009600-baae-4f5a-9f69-23b2bc916e4c')), RunInfo(run_id=UUID('acaf0838-242c-4255-85aa-8a62b675d046'))])" + "LLMResult(generations=[[Generation(text=\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\")], [Generation(text=\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\")]], llm_output={'model': 'meta-llama-3.1-8b-instruct'}, run=[RunInfo(run_id=UUID('ee97984b-6eab-4d40-a56f-51d6114953de')), RunInfo(run_id=UUID('cbe501ea-a20f-4420-9301-86cdfcf898c0'))], type='LLMResult')" ] }, "execution_count": 5, @@ -125,25 +140,19 @@ "output_type": "execute_result" } ], - "source": ["llm.generate([\"Tell me a joke.\", \"Tell me a joke.\"])"] + "source": [ + "llm.generate([\"Tell me a joke.\", \"Tell me a joke.\"])" + ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Username checks out.\n", - "User 1: I'm not sure if you're being sarcastic or not, but I'll take it as a compliment.\n", - "User 0: I'm not being sarcastic. I'm just saying that your username is very fitting.\n", - "User 1: Oh, I thought you were saying that I'm a \"dumbass\" because I'm a \"dumbass\" who \"checks out\"" - ] - } - ], - "source": ["for chunk in llm.stream(\"Tell me a joke.\"):\n print(chunk, end=\"\", flush=True)"] + "outputs": [], + "source": [ + "for chunk in llm.stream(\"Tell me a joke.\"):\n", + " print(chunk, end=\"\", flush=True)" + ] }, { "cell_type": "markdown", @@ -152,6 +161,26 @@ "You can also use all functionality of async APIs: `ainvoke`, `abatch`, `agenerate`, and `astream`." ] }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\"" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await llm.ainvoke(\"Tell me a joke.\")" + ] + }, { "cell_type": "code", "execution_count": 7, @@ -160,7 +189,8 @@ { "data": { "text/plain": [ - "'Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"'" + "[\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\",\n", + " \" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\"]" ] }, "execution_count": 7, @@ -168,7 +198,9 @@ "output_type": "execute_result" } ], - "source": ["await llm.ainvoke(\"Tell me a joke.\")"] + "source": [ + "await llm.abatch([\"Tell me a joke.\", \"Tell me a joke.\"])" + ] }, { "cell_type": "code", @@ -178,8 +210,7 @@ { "data": { "text/plain": [ - "['Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"',\n", - " 'Username checks out.\\nUser 1: I\\'m not sure if you\\'re being sarcastic or not, but I\\'ll take it as a compliment.\\nUser 0: I\\'m not being sarcastic. I\\'m just saying that your username is very fitting.\\nUser 1: Oh, I thought you were saying that I\\'m a \"dumbass\" because I\\'m a \"dumbass\" who \"checks out\"']" + "LLMResult(generations=[[Generation(text=\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\")], [Generation(text=\" I need a laugh.\\nHere's one: Why couldn't the bicycle stand up by itself?\\nBecause it was two-tired!\\nI hope that made you laugh! Do you want to hear another one? I have a million of 'em! (Okay, maybe not a million, but I have a few more where that came from!) What kind of joke are you in the mood for? A pun, a play on words, or something else? Let me know and I'll try to come\")]], llm_output={'model': 'meta-llama-3.1-8b-instruct'}, run=[RunInfo(run_id=UUID('857bd88e-e68a-46d2-8ad3-4a282c199a89')), RunInfo(run_id=UUID('a6ba6e7f-9a7a-4aa1-a2ac-c8fcf48309d3'))], type='LLMResult')" ] }, "execution_count": 8, @@ -187,48 +218,24 @@ "output_type": "execute_result" } ], - "source": ["await llm.abatch([\"Tell me a joke.\", \"Tell me a joke.\"])"] + "source": [ + "await llm.agenerate([\"Tell me a joke.\", \"Tell me a joke.\"])" + ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "LLMResult(generations=[[Generation(text=\"Username checks out.\\nUser 1: I'm not sure if you're being serious or not, but I'll take it as a compliment.\\nUser 0: I'm being serious. I'm not sure if you're being serious or not.\\nUser 1: I'm being serious. I'm not sure if you're being serious or not.\\nUser 0: I'm being serious. I'm not sure\")], [Generation(text=\"Username checks out.\\nUser 1: I'm not sure if you're being serious or not, but I'll take it as a compliment.\\nUser 0: I'm being serious. I'm not sure if you're being serious or not.\\nUser 1: I'm being serious. I'm not sure if you're being serious or not.\\nUser 0: I'm being serious. I'm not sure\")]], llm_output={'model': 'mixtral-8x7b-instruct-v0-1'}, run=[RunInfo(run_id=UUID('46144905-7350-4531-a4db-22e6a827c6e3')), RunInfo(run_id=UUID('e2b06c30-ffff-48cf-b792-be91f2144aa6'))])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": ["await llm.agenerate([\"Tell me a joke.\", \"Tell me a joke.\"])"] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Username checks out.\n", - "User 1: I'm not sure if you're being sarcastic or not, but I'll take it as a compliment.\n", - "User 0: I'm not being sarcastic. I'm just saying that your username is very fitting.\n", - "User 1: Oh, I thought you were saying that I'm a \"dumbass\" because I'm a \"dumbass\" who \"checks out\"" - ] - } - ], - "source": ["async for chunk in llm.astream(\"Tell me a joke.\"):\n print(chunk, end=\"\", flush=True)"] + "outputs": [], + "source": [ + "async for chunk in llm.astream(\"Tell me a joke.\"):\n", + " print(chunk, end=\"\", flush=True)" + ] } ], "metadata": { "kernelspec": { - "display_name": "langchain", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -242,7 +249,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/docs/docs/integrations/providers/friendli.md b/docs/docs/integrations/providers/friendli.mdx similarity index 56% rename from docs/docs/integrations/providers/friendli.md rename to docs/docs/integrations/providers/friendli.mdx index e0f3a49b68b..2e8fda9ade2 100644 --- a/docs/docs/integrations/providers/friendli.md +++ b/docs/docs/integrations/providers/friendli.mdx @@ -1,6 +1,6 @@ # Friendli AI ->[FriendliAI](https://friendli.ai/) enhances AI application performance and optimizes +> [FriendliAI](https://friendli.ai/) enhances AI application performance and optimizes > cost savings with scalable, efficient deployment options, tailored for high-demand AI workloads. ## Installation and setup @@ -8,10 +8,11 @@ Install the `friendli-client` python package. ```bash -pip install friendli-client +pip install -U langchain_community friendli-client ``` + Sign in to [Friendli Suite](https://suite.friendli.ai/) to create a Personal Access Token, -and set it as the `FRIENDLI_TOKEN` environment variable. +and set it as the `FRIENDLI_TOKEN` environment variabzle. ## Chat models @@ -20,6 +21,11 @@ See a [usage example](/docs/integrations/chat/friendli). ```python from langchain_community.chat_models.friendli import ChatFriendli + +chat = ChatFriendli(model='meta-llama-3.1-8b-instruct') + +for m in chat.stream("Tell me fun things to do in NYC"): + print(m.content, end="", flush=True) ``` ## LLMs @@ -28,4 +34,8 @@ See a [usage example](/docs/integrations/llms/friendli). ```python from langchain_community.llms.friendli import Friendli + +llm = Friendli(model='meta-llama-3.1-8b-instruct') + +print(llm.invoke("def bubble_sort(): ")) ``` diff --git a/docs/docs/integrations/providers/friendly.md b/docs/docs/integrations/providers/friendly.md deleted file mode 100644 index 834a1ebfe80..00000000000 --- a/docs/docs/integrations/providers/friendly.md +++ /dev/null @@ -1,32 +0,0 @@ -# Friendli AI - ->[Friendli AI](https://friendli.ai/) is a company that fine-tunes, deploys LLMs, -> and serves a wide range of Generative AI use cases. - - -## Installation and setup - -- Install the integration package: - - ``` - pip install friendli-client - ``` - -- Sign in to [Friendli Suite](https://suite.friendli.ai/) to create a Personal Access Token, -and set it as the `FRIENDLI_TOKEN` environment. - -## Chat models - -See a [usage example](/docs/integrations/chat/friendli). - -```python -from langchain_community.chat_models.friendli import ChatFriendli -``` - -## LLMs - -See a [usage example](/docs/integrations/llms/friendli). - -```python -from langchain_community.llms.friendli import Friendli -``` diff --git a/libs/community/langchain_community/chat_models/friendli.py b/libs/community/langchain_community/chat_models/friendli.py index a860ebf98bc..76bf70c9af1 100644 --- a/libs/community/langchain_community/chat_models/friendli.py +++ b/libs/community/langchain_community/chat_models/friendli.py @@ -75,12 +75,12 @@ class ChatFriendli(BaseChatModel, BaseFriendli): from langchain_community.chat_models import FriendliChat chat = Friendli( - model="llama-2-13b-chat", friendli_token="YOUR FRIENDLI TOKEN" + model="meta-llama-3.1-8b-instruct", friendli_token="YOUR FRIENDLI TOKEN" ) chat.invoke("What is generative AI?") """ - model: str = "llama-2-13b-chat" + model: str = "meta-llama-3.1-8b-instruct" @property def lc_secrets(self) -> Dict[str, str]: diff --git a/libs/community/langchain_community/llms/friendli.py b/libs/community/langchain_community/llms/friendli.py index 74b1ef7d199..d33c80eb39a 100644 --- a/libs/community/langchain_community/llms/friendli.py +++ b/libs/community/langchain_community/llms/friendli.py @@ -16,14 +16,19 @@ from langchain_core.utils.utils import convert_to_secret_str from pydantic import Field, SecretStr -def _stream_response_to_generation_chunk(stream_response: Any) -> GenerationChunk: +def _stream_response_to_generation_chunk( + stream_response: Any, +) -> GenerationChunk: """Convert a stream response to a generation chunk.""" - if stream_response.event == "token_sampled": - return GenerationChunk( - text=stream_response.text, - generation_info={"token": str(stream_response.token)}, - ) - return GenerationChunk(text="") + if not stream_response.get("choices", None): + return GenerationChunk(text="") + return GenerationChunk( + text=stream_response.choices[0].text, + # generation_info=dict( + # finish_reason=stream_response.choices[0].get("finish_reason", None), + # logprobs=stream_response.choices[0].get("logprobs", None), + # ), + ) class BaseFriendli(Serializable): @@ -34,7 +39,7 @@ class BaseFriendli(Serializable): # Friendli Async client. async_client: Any = Field(default=None, exclude=True) # Model name to use. - model: str = "mixtral-8x7b-instruct-v0-1" + model: str = "meta-llama-3.1-8b-instruct" # Friendli personal access token to run as. friendli_token: Optional[SecretStr] = None # Friendli team ID to run as. @@ -107,7 +112,7 @@ class Friendli(LLM, BaseFriendli): from langchain_community.llms import Friendli friendli = Friendli( - model="mixtral-8x7b-instruct-v0-1", friendli_token="YOUR FRIENDLI TOKEN" + model="meta-llama-3.1-8b-instruct", friendli_token="YOUR FRIENDLI TOKEN" ) """ diff --git a/libs/community/tests/unit_tests/llms/test_friendli.py b/libs/community/tests/unit_tests/llms/test_friendli.py index 6fd4593d93d..18cf5c33644 100644 --- a/libs/community/tests/unit_tests/llms/test_friendli.py +++ b/libs/community/tests/unit_tests/llms/test_friendli.py @@ -114,14 +114,14 @@ async def test_friendli_ainvoke( @pytest.mark.requires("friendli") def test_friendli_stream(mock_friendli_client: Mock, friendli_llm: Friendli) -> None: """Test stream with friendli.""" + mock_choice_0 = Mock() + mock_choice_0.text = "Hello " + mock_choice_1 = Mock() + mock_choice_1.text = "langchain" mock_chunk_0 = Mock() - mock_chunk_0.event = "token_sampled" - mock_chunk_0.text = "Hello " - mock_chunk_0.token = 0 + mock_chunk_0.choices = [mock_choice_0] mock_chunk_1 = Mock() - mock_chunk_1.event = "token_sampled" - mock_chunk_1.text = "Friendli" - mock_chunk_1.token = 1 + mock_chunk_1.choices = [mock_choice_1] mock_stream = MagicMock() mock_chunks = [mock_chunk_0, mock_chunk_1] mock_stream.__iter__.return_value = mock_chunks @@ -129,7 +129,7 @@ def test_friendli_stream(mock_friendli_client: Mock, friendli_llm: Friendli) -> mock_friendli_client.completions.create.return_value = mock_stream stream = friendli_llm.stream("Hello langchain") for i, chunk in enumerate(stream): - assert chunk == mock_chunks[i].text + assert chunk == mock_chunks[i].choices[0].text mock_friendli_client.completions.create.assert_called_once_with( model=friendli_llm.model, @@ -149,22 +149,22 @@ async def test_friendli_astream( mock_friendli_async_client: AsyncMock, friendli_llm: Friendli ) -> None: """Test async stream with friendli.""" + mock_choice_0 = Mock() + mock_choice_0.text = "Hello " + mock_choice_1 = Mock() + mock_choice_1.text = "langchain" mock_chunk_0 = Mock() - mock_chunk_0.event = "token_sampled" - mock_chunk_0.text = "Hello " - mock_chunk_0.token = 0 + mock_chunk_0.choices = [mock_choice_0] mock_chunk_1 = Mock() - mock_chunk_1.event = "token_sampled" - mock_chunk_1.text = "Friendli" - mock_chunk_1.token = 1 + mock_chunk_1.choices = [mock_choice_1] mock_stream = AsyncMock() mock_chunks = [mock_chunk_0, mock_chunk_1] - mock_stream.__aiter__.return_value = mock_chunks + mock_stream.__aiter__.return_value = iter(mock_chunks) mock_friendli_async_client.completions.create.return_value = mock_stream stream = friendli_llm.astream("Hello langchain") async for i, chunk in aenumerate(stream): - assert chunk == mock_chunks[i].text + assert chunk == mock_chunks[i].choices[0].text mock_friendli_async_client.completions.create.assert_awaited_once_with( model=friendli_llm.model,