From b09df253e306166906ca6fce10f5ecd693f6eadd Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Fri, 30 Jun 2023 12:48:59 -0700 Subject: [PATCH] maybe --- .../callbacks/integrations/promptlayer.ipynb | 216 ++++++++++++++++++ .../callbacks/integrations/promptlayer.ipynb | 2 - 2 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 docs/docs_skeleton/docs/modules/callbacks/integrations/promptlayer.ipynb diff --git a/docs/docs_skeleton/docs/modules/callbacks/integrations/promptlayer.ipynb b/docs/docs_skeleton/docs/modules/callbacks/integrations/promptlayer.ipynb new file mode 100644 index 00000000000..b326f034aab --- /dev/null +++ b/docs/docs_skeleton/docs/modules/callbacks/integrations/promptlayer.ipynb @@ -0,0 +1,216 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PromptLayer\n", + "\n", + "\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[PromptLayer](https://promptlayer.com) is a an observability platform for prompts and LLMs. In this guide we will go over how to setup the `PromptLayerCallbackHandler`. While PromptLayer does have LLMs that integrate directly with LangChain (eg [`PromptLayerOpenAI`](https://python.langchain.com/docs/modules/model_io/models/llms/integrations/promptlayer_openai)), this callback will be an easier and more feature rich way to integrate PromptLayer with any model on LangChain. \n", + "\n", + "This callback is also the recommended way to connect with PromptLayer when building Chains and Agents on LangChain." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Installation and Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install promptlayer --upgrade" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting API Credentials\n", + "\n", + "If you have not already create an account on [PromptLayer](https://www.promptlayer.com) and get an API key by clicking on the settings cog in the navbar\n", + "Set it as an environment variabled called `PROMPTLAYER_API_KEY`\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Usage\n", + "\n", + "To get started with `PromptLayerCallbackHandler` is fairly simple, it takes two optional arguments:\n", + "1. `pl_tags` - an optional list of strings that will be tags tracked on PromptLayer\n", + "2. `pl_id_callback` - an optional function that will get a `promptlayer_request_id` as an argument. This id can be used with all of PromptLayers tracking features to track, metadata, scores, and prompt usage." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Simple Example\n", + "\n", + "In this simple example we use `PromptLayerCallbackHandler` with `ChatOpenAI`. We add a PromptLayer tag named `chatopenai`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "content=\"Sure, here's one:\\n\\nWhy did the tomato turn red?\\n\\nBecause it saw the salad dressing!\" additional_kwargs={} example=False\n" + ] + } + ], + "source": [ + "from langchain.chat_models import ChatOpenAI\n", + "from langchain.schema import (\n", + " HumanMessage,\n", + ")\n", + "from langchain.callbacks import PromptLayerCallbackHandler\n", + "\n", + "chat_llm = ChatOpenAI(\n", + " temperature=0,\n", + " callbacks=[PromptLayerCallbackHandler(pl_tags=[\"chatopenai\"])],\n", + ")\n", + "llm_results = chat_llm(\n", + " [\n", + " HumanMessage(content=\"What comes after 1,2,3 ?\"),\n", + " HumanMessage(content=\"Tell me another joke?\"),\n", + " ]\n", + ")\n", + "print(llm_results)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Full Featured Example\n", + "\n", + "In this example we unlock more of the power of PromptLayer.\n", + "\n", + "We are using the Prompt Registry and fetching the prompt called `example`.\n", + "\n", + "We also define a `pl_id_callback` function that tracks a score, metadata and the prompt used. Read more about tracking on [our docs](docs.promptlayer.com)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "prompt layer id 6050929\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\nToasterCo.'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain.llms import OpenAI\n", + "from langchain.callbacks import PromptLayerCallbackHandler\n", + "import promptlayer\n", + "\n", + "def pl_id_callback(promptlayer_request_id):\n", + " print(\"prompt layer id \", promptlayer_request_id)\n", + " promptlayer.track.score(\n", + " request_id=promptlayer_request_id, score=100\n", + " ) # score is an integer 0-100\n", + " promptlayer.track.metadata(\n", + " request_id=promptlayer_request_id, metadata={\"foo\": \"bar\"}\n", + " ) # metadata is a dictionary of key value pairs that is tracked on PromptLayer\n", + " promptlayer.track.prompt(\n", + " request_id=promptlayer_request_id,\n", + " prompt_name=\"example\",\n", + " prompt_input_variables={\"product\": \"toasters\"},\n", + " version=1,\n", + " )\n", + "\n", + "\n", + "openai_llm = OpenAI(\n", + " model_name=\"text-davinci-002\",\n", + " callbacks=[PromptLayerCallbackHandler(pl_id_callback=pl_id_callback)],\n", + ")\n", + "\n", + "example_prompt = promptlayer.prompts.get(\"example\", version=1, langchain=True)\n", + "openai_llm(example_prompt.format(product=\"toasters\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That is all it takes! After setup all your requests will show up on the PromptLayer dasahboard.\n", + "This callback also works with any LLM implemented on LangChain." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + }, + "vscode": { + "interpreter": { + "hash": "c4fe2cd85a8d9e8baaec5340ce66faff1c77581a9f43e6c45e85e09b6fced008" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/extras/modules/callbacks/integrations/promptlayer.ipynb b/docs/extras/modules/callbacks/integrations/promptlayer.ipynb index 5527cb1980f..2b2f8e3f407 100644 --- a/docs/extras/modules/callbacks/integrations/promptlayer.ipynb +++ b/docs/extras/modules/callbacks/integrations/promptlayer.ipynb @@ -88,9 +88,7 @@ "source": [ "from langchain.chat_models import ChatOpenAI\n", "from langchain.schema import (\n", - " AIMessage,\n", " HumanMessage,\n", - " SystemMessage,\n", ")\n", "from langchain.callbacks import PromptLayerCallbackHandler\n", "\n",