mirror of
https://github.com/hwchase17/langchain.git
synced 2025-06-26 16:43:35 +00:00
community, docs: update token usage tracking callback + how-to guides (#22145)
This commit is contained in:
ccurme
GitHub
parent
2bc50fb895
commit
f39e1a2288
@ -14,35 +14,51 @@
|
||||
"\n",
|
||||
":::\n",
|
||||
"\n",
|
||||
"Tracking token usage to calculate cost is an important part of putting your app in production. This guide goes over how to obtain this information from your LangChain model calls."
|
||||
"Tracking token usage to calculate cost is an important part of putting your app in production. This guide goes over how to obtain this information from your LangChain model calls.\n",
|
||||
"\n",
|
||||
"This guide requires `langchain-openai >= 0.1.8`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9c7d1338-dd1b-4d06-b33d-d5cffc49fd6a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade --quiet langchain langchain-openai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1a55e87a-3291-4e7f-8e8e-4c69b0854384",
|
||||
"id": "598ae1e2-a52d-4459-81fd-cdc68b06742a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using AIMessage.response_metadata\n",
|
||||
"## Using LangSmith\n",
|
||||
"\n",
|
||||
"A number of model providers return token usage information as part of the chat generation response. When available, this is included in the [`AIMessage.response_metadata`](/docs/how_to/response_metadata) field. Here's an example with OpenAI:"
|
||||
"You can use [LangSmith](https://www.langchain.com/langsmith) to help track token usage in your LLM application. See the [LangSmith quick start guide](https://docs.smith.langchain.com/).\n",
|
||||
"\n",
|
||||
"## Using AIMessage.usage_metadata\n",
|
||||
"\n",
|
||||
"A number of model providers return token usage information as part of the chat generation response. When available, this information will be included on the `AIMessage` objects produced by the corresponding model.\n",
|
||||
"\n",
|
||||
"LangChain `AIMessage` objects include a [usage_metadata](https://api.python.langchain.com/en/latest/messages/langchain_core.messages.ai.AIMessage.html#langchain_core.messages.ai.AIMessage.usage_metadata) attribute. When populated, this attribute will be a [UsageMetadata](https://api.python.langchain.com/en/latest/messages/langchain_core.messages.ai.UsageMetadata.html) dictionary with standard keys (e.g., `\"input_tokens\"` and `\"output_tokens\"`).\n",
|
||||
"\n",
|
||||
"Examples:\n",
|
||||
"\n",
|
||||
"**OpenAI**:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "467ccdeb-6b62-45e5-816e-167cd24d2586",
|
||||
"id": "b39bf807-4125-4db4-bbf7-28a46afff6b4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'token_usage': {'completion_tokens': 225,\n",
|
||||
" 'prompt_tokens': 17,\n",
|
||||
" 'total_tokens': 242},\n",
|
||||
" 'model_name': 'gpt-4-turbo',\n",
|
||||
" 'system_fingerprint': 'fp_76f018034d',\n",
|
||||
" 'finish_reason': 'stop',\n",
|
||||
" 'logprobs': None}"
|
||||
"{'input_tokens': 8, 'output_tokens': 9, 'total_tokens': 17}"
|
||||
]
|
||||
},
|
||||
"execution_count": 1,
|
||||
@ -51,37 +67,33 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# !pip install -qU langchain-openai\n",
|
||||
"# # !pip install -qU langchain-openai\n",
|
||||
"\n",
|
||||
"from langchain_openai import ChatOpenAI\n",
|
||||
"\n",
|
||||
"llm = ChatOpenAI(model=\"gpt-4-turbo\")\n",
|
||||
"msg = llm.invoke([(\"human\", \"What's the oldest known example of cuneiform\")])\n",
|
||||
"msg.response_metadata"
|
||||
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\")\n",
|
||||
"openai_response = llm.invoke(\"hello\")\n",
|
||||
"openai_response.usage_metadata"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9d5026e9-3ad4-41e6-9946-9f1a26f4a21f",
|
||||
"id": "2299c44a-2fe6-4d52-a6a2-99ff6d231c73",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"And here's an example with Anthropic:"
|
||||
"**Anthropic**:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "145404f1-e088-4824-b468-236c486a9903",
|
||||
"id": "9c82ff80-ec4e-4049-b019-5f0bbd7df82a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'id': 'msg_01P61rdHbapEo6h3fjpfpCQT',\n",
|
||||
" 'model': 'claude-3-sonnet-20240229',\n",
|
||||
" 'stop_reason': 'end_turn',\n",
|
||||
" 'stop_sequence': None,\n",
|
||||
" 'usage': {'input_tokens': 17, 'output_tokens': 306}}"
|
||||
"{'input_tokens': 8, 'output_tokens': 12, 'total_tokens': 20}"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
@ -94,9 +106,222 @@
|
||||
"\n",
|
||||
"from langchain_anthropic import ChatAnthropic\n",
|
||||
"\n",
|
||||
"llm = ChatAnthropic(model=\"claude-3-sonnet-20240229\")\n",
|
||||
"msg = llm.invoke([(\"human\", \"What's the oldest known example of cuneiform\")])\n",
|
||||
"msg.response_metadata"
|
||||
"llm = ChatAnthropic(model=\"claude-3-haiku-20240307\")\n",
|
||||
"anthropic_response = llm.invoke(\"hello\")\n",
|
||||
"anthropic_response.usage_metadata"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6d4efc15-ba9f-4b3d-9278-8e01f99f263f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Using AIMessage.response_metadata\n",
|
||||
"\n",
|
||||
"Metadata from the model response is also included in the AIMessage [response_metadata](https://api.python.langchain.com/en/latest/messages/langchain_core.messages.ai.AIMessage.html#langchain_core.messages.ai.AIMessage.response_metadata) attribute. These data are typically not standardized. Note that different providers adopt different conventions for representing token counts:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "f156f9da-21f2-4c81-a714-54cbf9ad393e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"OpenAI: {'completion_tokens': 9, 'prompt_tokens': 8, 'total_tokens': 17}\n",
|
||||
"\n",
|
||||
"Anthropic: {'input_tokens': 8, 'output_tokens': 12}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(f'OpenAI: {openai_response.response_metadata[\"token_usage\"]}\\n')\n",
|
||||
"print(f'Anthropic: {anthropic_response.response_metadata[\"usage\"]}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b4ef2c43-0ff6-49eb-9782-e4070c9da8d7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Streaming\n",
|
||||
"\n",
|
||||
"Some providers support token count metadata in a streaming context.\n",
|
||||
"\n",
|
||||
"#### OpenAI\n",
|
||||
"\n",
|
||||
"For example, OpenAI will return a message [chunk](https://api.python.langchain.com/en/latest/messages/langchain_core.messages.ai.AIMessageChunk.html) at the end of a stream with token usage information. This behavior is supported by `langchain-openai >= 0.1.8` and can be enabled by setting `stream_options={\"include_usage\": True}`.\n",
|
||||
"\n",
|
||||
"```{=mdx}\n",
|
||||
":::note\n",
|
||||
"By default, the last message chunk in a stream will include a `\"finish_reason\"` in the message's `response_metadata` attribute. If we include token usage in streaming mode, an additional chunk containing usage metadata will be added to the end of the stream, such that `\"finish_reason\"` appears on the second to last message chunk.\n",
|
||||
":::\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "07f0c872-6b6c-4fed-a129-9b5a858505be",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"content='' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content='Hello' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content='!' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' How' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' can' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' I' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' assist' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' you' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content=' today' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content='?' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content='' response_metadata={'finish_reason': 'stop'} id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf'\n",
|
||||
"content='' id='run-b40e502e-d30e-4617-94ad-95b4dfee14bf' usage_metadata={'input_tokens': 8, 'output_tokens': 9, 'total_tokens': 17}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\")\n",
|
||||
"\n",
|
||||
"aggregate = None\n",
|
||||
"for chunk in llm.stream(\"hello\", stream_options={\"include_usage\": True}):\n",
|
||||
" print(chunk)\n",
|
||||
" aggregate = chunk if aggregate is None else aggregate + chunk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "dd809ded-8b13-4d5f-be5e-277b79d51802",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that the usage metadata will be included in the sum of the individual message chunks:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "3db7bc03-a7d4-4704-92ab-f8ba92ef59ae",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Hello! How can I assist you today?\n",
|
||||
"{'input_tokens': 8, 'output_tokens': 9, 'total_tokens': 17}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(aggregate.content)\n",
|
||||
"print(aggregate.usage_metadata)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7dba63e8-0ed7-4533-8f0f-78e19c38a25c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"To disable streaming token counts for OpenAI, set `\"include_usage\"` to False in `stream_options`, or omit it from the parameters:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "67117f2b-ce68-4c1e-9556-2d3849f90e1b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"content='' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content='Hello' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content='!' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' How' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' can' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' I' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' assist' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' you' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content=' today' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content='?' id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n",
|
||||
"content='' response_metadata={'finish_reason': 'stop'} id='run-0085d64c-13d2-431b-a0fa-399be8cd3c52'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"aggregate = None\n",
|
||||
"for chunk in llm.stream(\"hello\"):\n",
|
||||
" print(chunk)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6a5d9617-be3a-419a-9276-de9c29fa50ae",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You can also enable streaming token usage by setting `model_kwargs` when instantiating the chat model. This can be useful when incorporating chat models into LangChain [chains](/docs/concepts#langchain-expression-language-lcel): usage metadata can be monitored when [streaming intermediate steps](/docs/how_to/streaming#using-stream-events) or using tracing software such as [LangSmith](https://docs.smith.langchain.com/).\n",
|
||||
"\n",
|
||||
"See the below example, where we return output structured to a desired schema, but can still observe token usage streamed from intermediate steps."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "57dec1fb-bd9c-4c98-8798-8fbbe67f6b2c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Token usage: {'input_tokens': 79, 'output_tokens': 23, 'total_tokens': 102}\n",
|
||||
"\n",
|
||||
"setup='Why was the math book sad?' punchline='Because it had too many problems.'\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class Joke(BaseModel):\n",
|
||||
" \"\"\"Joke to tell user.\"\"\"\n",
|
||||
"\n",
|
||||
" setup: str = Field(description=\"question to set up a joke\")\n",
|
||||
" punchline: str = Field(description=\"answer to resolve the joke\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"llm = ChatOpenAI(\n",
|
||||
" model=\"gpt-3.5-turbo-0125\",\n",
|
||||
" model_kwargs={\"stream_options\": {\"include_usage\": True}},\n",
|
||||
")\n",
|
||||
"# Under the hood, .with_structured_output binds tools to the\n",
|
||||
"# chat model and appends a parser.\n",
|
||||
"structured_llm = llm.with_structured_output(Joke)\n",
|
||||
"\n",
|
||||
"async for event in structured_llm.astream_events(\"Tell me a joke\", version=\"v2\"):\n",
|
||||
" if event[\"event\"] == \"on_chat_model_end\":\n",
|
||||
" print(f'Token usage: {event[\"data\"][\"output\"].usage_metadata}\\n')\n",
|
||||
" elif event[\"event\"] == \"on_chain_end\":\n",
|
||||
" print(event[\"data\"][\"output\"])\n",
|
||||
" else:\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2bc8d313-4bef-463e-89a5-236d8bb6ab2f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Token usage is also visible in the corresponding [LangSmith trace](https://smith.langchain.com/public/fe6513d5-7212-4045-82e0-fefa28bc7656/r) in the payload from the chat model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -115,7 +340,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 9,
|
||||
"id": "31667d54",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -123,11 +348,11 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tokens Used: 26\n",
|
||||
"Tokens Used: 27\n",
|
||||
"\tPrompt Tokens: 11\n",
|
||||
"\tCompletion Tokens: 15\n",
|
||||
"\tCompletion Tokens: 16\n",
|
||||
"Successful Requests: 1\n",
|
||||
"Total Cost (USD): $0.00056\n"
|
||||
"Total Cost (USD): $2.95e-05\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -136,7 +361,7 @@
|
||||
"\n",
|
||||
"from langchain_community.callbacks.manager import get_openai_callback\n",
|
||||
"\n",
|
||||
"llm = ChatOpenAI(model=\"gpt-4-turbo\", temperature=0)\n",
|
||||
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
|
||||
"\n",
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" result = llm.invoke(\"Tell me a joke\")\n",
|
||||
@ -153,7 +378,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 10,
|
||||
"id": "e09420f4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -161,7 +386,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"52\n"
|
||||
"55\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -172,6 +397,39 @@
|
||||
" print(cb.total_tokens)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9ac51188-c8f4-4230-90fd-3cd78cdd955d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"```{=mdx}\n",
|
||||
":::note\n",
|
||||
"Cost information is currently not available in streaming mode. This is because model names are currently not propagated through chunks in streaming mode, and the model name is used to look up the correct pricing. Token counts however are available:\n",
|
||||
":::\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "b241069a-265d-4497-af34-b0a5f95ae67f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"28\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" for chunk in llm.stream(\"Tell me a joke\", stream_options={\"include_usage\": True}):\n",
|
||||
" pass\n",
|
||||
" print(cb.total_tokens)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d8186e7b",
|
||||
@ -182,7 +440,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 12,
|
||||
"id": "5d1125c6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -211,15 +469,15 @@
|
||||
"source": [
|
||||
"```{=mdx}\n",
|
||||
":::note\n",
|
||||
"We have to set `stream_runnable=False` for token counting to work. By default the AgentExecutor will stream the underlying agent so that you can get the most granular results when streaming events via AgentExecutor.stream_events. However, OpenAI does not return token counts when streaming model responses, so we need to turn off the underlying streaming.\n",
|
||||
"We have to set `stream_runnable=False` for cost information, as described above. By default the AgentExecutor will stream the underlying agent so that you can get the most granular results when streaming events via AgentExecutor.stream_events.\n",
|
||||
":::\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "2f98c536",
|
||||
"execution_count": 13,
|
||||
"id": "3950d88b-8bfb-4294-b75b-e6fd421e633c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -230,46 +488,51 @@
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `wikipedia` with `Hummingbird`\n",
|
||||
"Invoking: `wikipedia` with `{'query': 'hummingbird scientific name'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mPage: Hummingbird\n",
|
||||
"Summary: Hummingbirds are birds native to the Americas and comprise the biological family Trochilidae. With approximately 366 species and 113 genera, they occur from Alaska to Tierra del Fuego, but most species are found in Central and South America. As of 2024, 21 hummingbird species are listed as endangered or critically endangered, with numerous species declining in population.Hummingbirds have varied specialized characteristics to enable rapid, maneuverable flight: exceptional metabolic capacity, adaptations to high altitude, sensitive visual and communication abilities, and long-distance migration in some species. Among all birds, male hummingbirds have the widest diversity of plumage color, particularly in blues, greens, and purples. Hummingbirds are the smallest mature birds, measuring 7.5–13 cm (3–5 in) in length. The smallest is the 5 cm (2.0 in) bee hummingbird, which weighs less than 2.0 g (0.07 oz), and the largest is the 23 cm (9 in) giant hummingbird, weighing 18–24 grams (0.63–0.85 oz). Noted for long beaks, hummingbirds are specialized for feeding on flower nectar, but all species also consume small insects.\n",
|
||||
"Summary: Hummingbirds are birds native to the Americas and comprise the biological family Trochilidae. With approximately 366 species and 113 genera, they occur from Alaska to Tierra del Fuego, but most species are found in Central and South America. As of 2024, 21 hummingbird species are listed as endangered or critically endangered, with numerous species declining in population.\n",
|
||||
"Hummingbirds have varied specialized characteristics to enable rapid, maneuverable flight: exceptional metabolic capacity, adaptations to high altitude, sensitive visual and communication abilities, and long-distance migration in some species. Among all birds, male hummingbirds have the widest diversity of plumage color, particularly in blues, greens, and purples. Hummingbirds are the smallest mature birds, measuring 7.5–13 cm (3–5 in) in length. The smallest is the 5 cm (2.0 in) bee hummingbird, which weighs less than 2.0 g (0.07 oz), and the largest is the 23 cm (9 in) giant hummingbird, weighing 18–24 grams (0.63–0.85 oz). Noted for long beaks, hummingbirds are specialized for feeding on flower nectar, but all species also consume small insects.\n",
|
||||
"They are known as hummingbirds because of the humming sound created by their beating wings, which flap at high frequencies audible to other birds and humans. They hover at rapid wing-flapping rates, which vary from around 12 beats per second in the largest species to 80 per second in small hummingbirds.\n",
|
||||
"Hummingbirds have the highest mass-specific metabolic rate of any homeothermic animal. To conserve energy when food is scarce and at night when not foraging, they can enter torpor, a state similar to hibernation, and slow their metabolic rate to 1⁄15 of its normal rate. While most hummingbirds do not migrate, the rufous hummingbird has one of the longest migrations among birds, traveling twice per year between Alaska and Mexico, a distance of about 3,900 miles (6,300 km).\n",
|
||||
"Hummingbirds split from their sister group, the swifts and treeswifts, around 42 million years ago. The oldest known fossil hummingbird is Eurotrochilus, from the Rupelian Stage of Early Oligocene Europe.\n",
|
||||
"\n",
|
||||
"Page: Rufous hummingbird\n",
|
||||
"Summary: The rufous hummingbird (Selasphorus rufus) is a small hummingbird, about 8 cm (3.1 in) long with a long, straight and slender bill. These birds are known for their extraordinary flight skills, flying 2,000 mi (3,200 km) during their migratory transits. It is one of nine species in the genus Selasphorus.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Page: Bee hummingbird\n",
|
||||
"Summary: The bee hummingbird, zunzuncito or Helena hummingbird (Mellisuga helenae) is a species of hummingbird, native to the island of Cuba in the Caribbean. It is the smallest known bird. The bee hummingbird feeds on nectar of flowers and bugs found in Cuba.\n",
|
||||
"\n",
|
||||
"Page: Hummingbird cake\n",
|
||||
"Summary: Hummingbird cake is a banana-pineapple spice cake originating in Jamaica and a popular dessert in the southern United States since the 1970s. Ingredients include flour, sugar, salt, vegetable oil, ripe banana, pineapple, cinnamon, pecans, vanilla extract, eggs, and leavening agent. It is often served with cream cheese frosting.\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `wikipedia` with `Fastest bird`\n",
|
||||
"Page: Anna's hummingbird\n",
|
||||
"Summary: Anna's hummingbird (Calypte anna) is a North American species of hummingbird. It was named after Anna Masséna, Duchess of Rivoli.\n",
|
||||
"It is native to western coastal regions of North America. In the early 20th century, Anna's hummingbirds bred only in northern Baja California and Southern California. The transplanting of exotic ornamental plants in residential areas throughout the Pacific coast and inland deserts provided expanded nectar and nesting sites, allowing the species to expand its breeding range. Year-round residence of Anna's hummingbirds in the Pacific Northwest is an example of ecological release dependent on acclimation to colder winter temperatures, introduced plants, and human provision of nectar feeders during winter.\n",
|
||||
"These birds feed on nectar from flowers using a long extendable tongue. They also consume small insects and other arthropods caught in flight or gleaned from vegetation.\u001b[0m\u001b[32;1m\u001b[1;3m\n",
|
||||
"Invoking: `wikipedia` with `{'query': 'fastest bird species'}`\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mPage: Fastest animals\n",
|
||||
"\u001b[0m\u001b[36;1m\u001b[1;3mPage: List of birds by flight speed\n",
|
||||
"Summary: This is a list of the fastest flying birds in the world. A bird's velocity is necessarily variable; a hunting bird will reach much greater speeds while diving to catch prey than when flying horizontally. The bird that can achieve the greatest airspeed is the peregrine falcon (Falco peregrinus), able to exceed 320 km/h (200 mph) in its dives. A close relative of the common swift, the white-throated needletail (Hirundapus caudacutus), is commonly reported as the fastest bird in level flight with a reported top speed of 169 km/h (105 mph). This record remains unconfirmed as the measurement methods have never been published or verified. The record for the fastest confirmed level flight by a bird is 111.5 km/h (69.3 mph) held by the common swift.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Page: Fastest animals\n",
|
||||
"Summary: This is a list of the fastest animals in the world, by types of animal.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Page: List of birds by flight speed\n",
|
||||
"Summary: This is a list of the fastest flying birds in the world. A bird's velocity is necessarily variable; a hunting bird will reach much greater speeds while diving to catch prey than when flying horizontally. The bird that can achieve the greatest airspeed is the peregrine falcon, able to exceed 320 km/h (200 mph) in its dives. A close relative of the common swift, the white-throated needletail (Hirundapus caudacutus), is commonly reported as the fastest bird in level flight with a reported top speed of 169 km/h (105 mph). This record remains unconfirmed as the measurement methods have never been published or verified. The record for the fastest confirmed level flight by a bird is 111.5 km/h (69.3 mph) held by the common swift.\n",
|
||||
"\n",
|
||||
"Page: Ostrich\n",
|
||||
"Summary: Ostriches are large flightless birds. They are the heaviest and largest living birds, with adult common ostriches weighing anywhere between 63.5 and 145 kilograms and laying the largest eggs of any living land animal. With the ability to run at 70 km/h (43.5 mph), they are the fastest birds on land. They are farmed worldwide, with significant industries in the Philippines and in Namibia. Ostrich leather is a lucrative commodity, and the large feathers are used as plumes for the decoration of ceremonial headgear. Ostrich eggs have been used by humans for millennia.\n",
|
||||
"Ostriches are of the genus Struthio in the order Struthioniformes, part of the infra-class Palaeognathae, a diverse group of flightless birds also known as ratites that includes the emus, rheas, cassowaries, kiwis and the extinct elephant birds and moas. There are two living species of ostrich: the common ostrich, native to large areas of sub-Saharan Africa, and the Somali ostrich, native to the Horn of Africa. The common ostrich was historically native to the Arabian Peninsula, and ostriches were present across Asia as far east as China and Mongolia during the Late Pleistocene and possibly into the Holocene.\u001b[0m\u001b[32;1m\u001b[1;3m### Hummingbird's Scientific Name\n",
|
||||
"The scientific name for the bee hummingbird, which is the smallest known bird and a species of hummingbird, is **Mellisuga helenae**. It is native to Cuba.\n",
|
||||
"\n",
|
||||
"### Fastest Bird Species\n",
|
||||
"The fastest bird in terms of airspeed is the **peregrine falcon**, which can exceed speeds of 320 km/h (200 mph) during its diving flight. In level flight, the fastest confirmed speed is held by the **common swift**, which can fly at 111.5 km/h (69.3 mph).\u001b[0m\n",
|
||||
"Page: Falcon\n",
|
||||
"Summary: Falcons () are birds of prey in the genus Falco, which includes about 40 species. Falcons are widely distributed on all continents of the world except Antarctica, though closely related raptors did occur there in the Eocene.\n",
|
||||
"Adult falcons have thin, tapered wings, which enable them to fly at high speed and change direction rapidly. Fledgling falcons, in their first year of flying, have longer flight feathers, which make their configuration more like that of a general-purpose bird such as a broad wing. This makes flying easier while learning the exceptional skills required to be effective hunters as adults.\n",
|
||||
"The falcons are the largest genus in the Falconinae subfamily of Falconidae, which itself also includes another subfamily comprising caracaras and a few other species. All these birds kill with their beaks, using a tomial \"tooth\" on the side of their beaks—unlike the hawks, eagles, and other birds of prey in the Accipitridae, which use their feet.\n",
|
||||
"The largest falcon is the gyrfalcon at up to 65 cm in length. The smallest falcon species is the pygmy falcon, which measures just 20 cm. As with hawks and owls, falcons exhibit sexual dimorphism, with the females typically larger than the males, thus allowing a wider range of prey species.\n",
|
||||
"Some small falcons with long, narrow wings are called \"hobbies\" and some which hover while hunting are called \"kestrels\".\n",
|
||||
"As is the case with many birds of prey, falcons have exceptional powers of vision; the visual acuity of one species has been measured at 2.6 times that of a normal human. Peregrine falcons have been recorded diving at speeds of 320 km/h (200 mph), making them the fastest-moving creatures on Earth; the fastest recorded dive attained a vertical speed of 390 km/h (240 mph).\u001b[0m\u001b[32;1m\u001b[1;3mThe scientific name for a hummingbird is Trochilidae. The fastest bird species is the peregrine falcon (Falco peregrinus), which can exceed speeds of 320 km/h (200 mph) in its dives.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n",
|
||||
"Total Tokens: 1583\n",
|
||||
"Prompt Tokens: 1412\n",
|
||||
"Completion Tokens: 171\n",
|
||||
"Total Cost (USD): $0.019250000000000003\n"
|
||||
"Total Tokens: 1787\n",
|
||||
"Prompt Tokens: 1687\n",
|
||||
"Completion Tokens: 100\n",
|
||||
"Total Cost (USD): $0.0009935\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -298,19 +561,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "4a3eced5-2ff7-49a7-a48b-768af8658323",
|
||||
"execution_count": 12,
|
||||
"id": "1837c807-136a-49d8-9c33-060e58dc16d2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tokens Used: 0\n",
|
||||
"\tPrompt Tokens: 0\n",
|
||||
"\tCompletion Tokens: 0\n",
|
||||
"Tokens Used: 96\n",
|
||||
"\tPrompt Tokens: 26\n",
|
||||
"\tCompletion Tokens: 70\n",
|
||||
"Successful Requests: 2\n",
|
||||
"Total Cost (USD): $0.0\n"
|
||||
"Total Cost (USD): $0.001888\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -364,7 +627,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.1"
|
||||
"version": "3.10.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@ -2,156 +2,85 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e5715368",
|
||||
"id": "90dff237-bc28-4185-a2c0-d5203bbdeacd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# How to track token usage for LLMs\n",
|
||||
"\n",
|
||||
"This notebook goes over how to track your token usage for specific calls. It is currently only implemented for the OpenAI API.\n",
|
||||
"Tracking token usage to calculate cost is an important part of putting your app in production. This guide goes over how to obtain this information from your LangChain model calls.\n",
|
||||
"\n",
|
||||
"Let's first look at an extremely simple example of tracking token usage for a single LLM call."
|
||||
":::info Prerequisites\n",
|
||||
"\n",
|
||||
"This guide assumes familiarity with the following concepts:\n",
|
||||
"\n",
|
||||
"- [LLMs](/docs/concepts/#llms)\n",
|
||||
":::\n",
|
||||
"\n",
|
||||
"## Using LangSmith\n",
|
||||
"\n",
|
||||
"You can use [LangSmith](https://www.langchain.com/langsmith) to help track token usage in your LLM application. See the [LangSmith quick start guide](https://docs.smith.langchain.com/).\n",
|
||||
"\n",
|
||||
"## Using callbacks\n",
|
||||
"\n",
|
||||
"There are some API-specific callback context managers that allow you to track token usage across multiple calls. You'll need to check whether such an integration is available for your particular model.\n",
|
||||
"\n",
|
||||
"If such an integration is not available for your model, you can create a custom callback manager by adapting the implementation of the [OpenAI callback manager](https://api.python.langchain.com/en/latest/_modules/langchain_community/callbacks/openai_info.html#OpenAICallbackHandler).\n",
|
||||
"\n",
|
||||
"### OpenAI\n",
|
||||
"\n",
|
||||
"Let's first look at an extremely simple example of tracking token usage for a single Chat model call.\n",
|
||||
"\n",
|
||||
":::{.callout-danger}\n",
|
||||
"\n",
|
||||
"The callback handler does not currently support streaming token counts for legacy language models (e.g., `langchain_openai.OpenAI`). For support in a streaming context, refer to the corresponding guide for chat models [here](/docs/how_to/chat_token_usage_tracking).\n",
|
||||
"\n",
|
||||
":::"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f790edd9-823e-4bc5-befa-e9529c7237a0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Single call"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "9455db35",
|
||||
"id": "2eebbee2-6ca1-4fa8-a3aa-0376888ceefb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"Why don't scientists trust atoms?\n",
|
||||
"\n",
|
||||
"Because they make up everything.\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"Total Tokens: 18\n",
|
||||
"Prompt Tokens: 4\n",
|
||||
"Completion Tokens: 14\n",
|
||||
"Total Cost (USD): $3.4e-05\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_community.callbacks import get_openai_callback\n",
|
||||
"from langchain_openai import OpenAI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "d1c55cc9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"llm = OpenAI(model_name=\"gpt-3.5-turbo-instruct\", n=2, best_of=2)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "31667d54",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tokens Used: 37\n",
|
||||
"\tPrompt Tokens: 4\n",
|
||||
"\tCompletion Tokens: 33\n",
|
||||
"Successful Requests: 1\n",
|
||||
"Total Cost (USD): $7.2e-05\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" result = llm.invoke(\"Tell me a joke\")\n",
|
||||
" print(cb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0ab6d27",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Anything inside the context manager will get tracked. Here's an example of using it to track multiple calls in sequence."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "e09420f4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"72\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" result = llm.invoke(\"Tell me a joke\")\n",
|
||||
" result2 = llm.invoke(\"Tell me a joke\")\n",
|
||||
" print(cb.total_tokens)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d8186e7b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"If a chain or agent with multiple steps in it is used, it will track all those steps."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "5d1125c6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.agents import AgentType, initialize_agent, load_tools\n",
|
||||
"from langchain_openai import OpenAI\n",
|
||||
"\n",
|
||||
"llm = OpenAI(temperature=0)\n",
|
||||
"tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm)\n",
|
||||
"agent = initialize_agent(\n",
|
||||
" tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "2f98c536",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"llm = OpenAI(model_name=\"gpt-3.5-turbo-instruct\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
|
||||
"\u001b[32;1m\u001b[1;3m I need to find out who Olivia Wilde's boyfriend is and then calculate his age raised to the 0.23 power.\n",
|
||||
"Action: Search\n",
|
||||
"Action Input: \"Olivia Wilde boyfriend\"\u001b[0m\n",
|
||||
"Observation: \u001b[36;1m\u001b[1;3m[\"Olivia Wilde and Harry Styles took fans by surprise with their whirlwind romance, which began when they met on the set of Don't Worry Darling.\", 'Olivia Wilde started dating Harry Styles after ending her years-long engagement to Jason Sudeikis — see their relationship timeline.', 'Olivia Wilde and Harry Styles were spotted early on in their relationship walking around London. (. Image ...', \"Looks like Olivia Wilde and Jason Sudeikis are starting 2023 on good terms. Amid their highly publicized custody battle – and the actress' ...\", 'The two started dating after Wilde split up with actor Jason Sudeikisin 2020. However, their relationship came to an end last November.', \"Olivia Wilde and Harry Styles started dating during the filming of Don't Worry Darling. While the movie got a lot of backlash because of the ...\", \"Here's what we know so far about Harry Styles and Olivia Wilde's relationship.\", 'Olivia and the Grammy winner kept their romance out of the spotlight as their relationship began just two months after her split from ex-fiancé ...', \"Harry Styles and Olivia Wilde first met on the set of Don't Worry Darling and stepped out as a couple in January 2021. Relive all their biggest relationship ...\"]\u001b[0m\n",
|
||||
"Thought:\u001b[32;1m\u001b[1;3m Harry Styles is Olivia Wilde's boyfriend.\n",
|
||||
"Action: Search\n",
|
||||
"Action Input: \"Harry Styles age\"\u001b[0m\n",
|
||||
"Observation: \u001b[36;1m\u001b[1;3m29 years\u001b[0m\n",
|
||||
"Thought:\u001b[32;1m\u001b[1;3m I need to calculate 29 raised to the 0.23 power.\n",
|
||||
"Action: Calculator\n",
|
||||
"Action Input: 29^0.23\u001b[0m\n",
|
||||
"Observation: \u001b[33;1m\u001b[1;3mAnswer: 2.169459462491557\u001b[0m\n",
|
||||
"Thought:\u001b[32;1m\u001b[1;3m I now know the final answer.\n",
|
||||
"Final Answer: Harry Styles is Olivia Wilde's boyfriend and his current age raised to the 0.23 power is 2.169459462491557.\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m> Finished chain.\u001b[0m\n",
|
||||
"Total Tokens: 2205\n",
|
||||
"Prompt Tokens: 2053\n",
|
||||
"Completion Tokens: 152\n",
|
||||
"Total Cost (USD): $0.0441\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" response = agent.run(\n",
|
||||
" \"Who is Olivia Wilde's boyfriend? What is his current age raised to the 0.23 power?\"\n",
|
||||
" )\n",
|
||||
" result = llm.invoke(\"Tell me a joke\")\n",
|
||||
" print(result)\n",
|
||||
" print(\"---\")\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(f\"Total Tokens: {cb.total_tokens}\")\n",
|
||||
"print(f\"Prompt Tokens: {cb.prompt_tokens}\")\n",
|
||||
"print(f\"Completion Tokens: {cb.completion_tokens}\")\n",
|
||||
@ -159,12 +88,140 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "80ca77a3",
|
||||
"cell_type": "markdown",
|
||||
"id": "7df3be35-dd97-4e3a-bd51-52434ab2249d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"### Multiple calls\n",
|
||||
"\n",
|
||||
"Anything inside the context manager will get tracked. Here's an example of using it to track multiple calls in sequence to a chain. This will also work for an agent which may use multiple steps."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "3ec10419-294c-44bf-af85-86aabf457cb6",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"Why did the chicken go to the seance?\n",
|
||||
"\n",
|
||||
"To talk to the other side of the road!\n",
|
||||
"--\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Why did the fish need a lawyer?\n",
|
||||
"\n",
|
||||
"Because it got caught in a net!\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"Total Tokens: 50\n",
|
||||
"Prompt Tokens: 12\n",
|
||||
"Completion Tokens: 38\n",
|
||||
"Total Cost (USD): $9.400000000000001e-05\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_community.callbacks import get_openai_callback\n",
|
||||
"from langchain_core.prompts import PromptTemplate\n",
|
||||
"from langchain_openai import OpenAI\n",
|
||||
"\n",
|
||||
"llm = OpenAI(model_name=\"gpt-3.5-turbo-instruct\")\n",
|
||||
"\n",
|
||||
"template = PromptTemplate.from_template(\"Tell me a joke about {topic}\")\n",
|
||||
"chain = template | llm\n",
|
||||
"\n",
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" response = chain.invoke({\"topic\": \"birds\"})\n",
|
||||
" print(response)\n",
|
||||
" response = chain.invoke({\"topic\": \"fish\"})\n",
|
||||
" print(\"--\")\n",
|
||||
" print(response)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print()\n",
|
||||
"print(\"---\")\n",
|
||||
"print(f\"Total Tokens: {cb.total_tokens}\")\n",
|
||||
"print(f\"Prompt Tokens: {cb.prompt_tokens}\")\n",
|
||||
"print(f\"Completion Tokens: {cb.completion_tokens}\")\n",
|
||||
"print(f\"Total Cost (USD): ${cb.total_cost}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ad7a3fba-9fac-4222-8f87-d1d276d27d6e",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"source": [
|
||||
"## Streaming\n",
|
||||
"\n",
|
||||
":::{.callout-danger}\n",
|
||||
"\n",
|
||||
"`get_openai_callback` does not currently support streaming token counts for legacy language models (e.g., `langchain_openai.OpenAI`). If you want to count tokens correctly in a streaming context, there are a number of options:\n",
|
||||
"\n",
|
||||
"- Use chat models as described in [this guide](/docs/how_to/chat_token_usage_tracking);\n",
|
||||
"- Implement a [custom callback handler](/docs/how_to/custom_callbacks/) that uses appropriate tokenizers to count the tokens;\n",
|
||||
"- Use a monitoring platform such as [LangSmith](https://www.langchain.com/langsmith).\n",
|
||||
":::\n",
|
||||
"\n",
|
||||
"Note that when using legacy language models in a streaming context, token counts are not updated:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "cd61ed79-7858-49bb-afb5-d41291f597ba",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
"Why don't scientists trust atoms?\n",
|
||||
"\n",
|
||||
"Because they make up everything!\n",
|
||||
"\n",
|
||||
"Why don't scientists trust atoms?\n",
|
||||
"\n",
|
||||
"Because they make up everything.\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"Total Tokens: 0\n",
|
||||
"Prompt Tokens: 0\n",
|
||||
"Completion Tokens: 0\n",
|
||||
"Total Cost (USD): $0.0\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from langchain_community.callbacks import get_openai_callback\n",
|
||||
"from langchain_openai import OpenAI\n",
|
||||
"\n",
|
||||
"llm = OpenAI(model_name=\"gpt-3.5-turbo-instruct\")\n",
|
||||
"\n",
|
||||
"with get_openai_callback() as cb:\n",
|
||||
" for chunk in llm.stream(\"Tell me a joke\"):\n",
|
||||
" print(chunk, end=\"\", flush=True)\n",
|
||||
" print(result)\n",
|
||||
" print(\"---\")\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(f\"Total Tokens: {cb.total_tokens}\")\n",
|
||||
"print(f\"Prompt Tokens: {cb.prompt_tokens}\")\n",
|
||||
"print(f\"Completion Tokens: {cb.completion_tokens}\")\n",
|
||||
"print(f\"Total Cost (USD): ${cb.total_cost}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
@ -183,7 +240,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.1"
|
||||
"version": "3.10.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@ -4,7 +4,8 @@ import threading
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from langchain_core.callbacks import BaseCallbackHandler
|
||||
from langchain_core.outputs import LLMResult
|
||||
from langchain_core.messages import AIMessage
|
||||
from langchain_core.outputs import ChatGeneration, LLMResult
|
||||
|
||||
MODEL_COST_PER_1K_TOKENS = {
|
||||
# GPT-4o input
|
||||
@ -210,6 +211,36 @@ class OpenAICallbackHandler(BaseCallbackHandler):
|
||||
|
||||
def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
|
||||
"""Collect token usage."""
|
||||
# Check for usage_metadata (langchain-core >= 0.2.2)
|
||||
try:
|
||||
generation = response.generations[0][0]
|
||||
except IndexError:
|
||||
generation = None
|
||||
if isinstance(generation, ChatGeneration):
|
||||
try:
|
||||
message = generation.message
|
||||
if isinstance(message, AIMessage):
|
||||
usage_metadata = message.usage_metadata
|
||||
else:
|
||||
usage_metadata = None
|
||||
except AttributeError:
|
||||
usage_metadata = None
|
||||
else:
|
||||
usage_metadata = None
|
||||
if usage_metadata:
|
||||
token_usage = {"total_tokens": usage_metadata["total_tokens"]}
|
||||
completion_tokens = usage_metadata["output_tokens"]
|
||||
prompt_tokens = usage_metadata["input_tokens"]
|
||||
if response.llm_output is None:
|
||||
# model name (and therefore cost) is unavailable in
|
||||
# streaming responses
|
||||
model_name = ""
|
||||
else:
|
||||
model_name = standardize_model_name(
|
||||
response.llm_output.get("model_name", "")
|
||||
)
|
||||
|
||||
else:
|
||||
if response.llm_output is None:
|
||||
return None
|
||||
|
||||
@ -222,7 +253,9 @@ class OpenAICallbackHandler(BaseCallbackHandler):
|
||||
token_usage = response.llm_output["token_usage"]
|
||||
completion_tokens = token_usage.get("completion_tokens", 0)
|
||||
prompt_tokens = token_usage.get("prompt_tokens", 0)
|
||||
model_name = standardize_model_name(response.llm_output.get("model_name", ""))
|
||||
model_name = standardize_model_name(
|
||||
response.llm_output.get("model_name", "")
|
||||
)
|
||||
if model_name in MODEL_COST_PER_1K_TOKENS:
|
||||
completion_cost = get_openai_token_cost_for_model(
|
||||
model_name, completion_tokens, is_completion=True
|
||||
|
||||
Loading…
Reference in New Issue
Block a user