diff --git a/docs/docs/integrations/callbacks/uptrain.ipynb b/docs/docs/integrations/callbacks/uptrain.ipynb index 0dbb04f9020..d69441fc7b7 100644 --- a/docs/docs/integrations/callbacks/uptrain.ipynb +++ b/docs/docs/integrations/callbacks/uptrain.ipynb @@ -58,9 +58,28 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: There was an error checking the latest version of pip.\u001b[0m\u001b[33m\n", + "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU langchain langchain_openai uptrain faiss-cpu flashrank" ] @@ -81,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -146,7 +165,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -164,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -181,7 +200,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -245,9 +264,31 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-04-17 17:03:44.969\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m378\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", + "\u001b[32m2024-04-17 17:04:05.809\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m367\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Question: What did the president say about Ketanji Brown Jackson\n", + "Response: The president mentioned that he had nominated Ketanji Brown Jackson to serve on the United States Supreme Court 4 days ago. He described her as one of the nation's top legal minds who will continue Justice Breyer’s legacy of excellence. He also mentioned that she is a former top litigator in private practice, a former federal public defender, and comes from a family of public school educators and police officers. He described her as a consensus builder and noted that since her nomination, she has received a broad range of support from various groups, including the Fraternal Order of Police and former judges appointed by both Democrats and Republicans.\n", + "\n", + "Context Relevance Score: 1.0\n", + "Factual Accuracy Score: 1.0\n", + "Response Completeness Score: 1.0\n" + ] + } + ], "source": [ "# Create the RAG prompt\n", "template = \"\"\"Answer the question based only on the following context, which can include text and tables:\n", @@ -287,15 +328,15 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-04-10 14:09:15.887\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m376\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", - "\u001b[32m2024-04-10 14:09:21.367\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m365\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" + "\u001b[32m2024-04-17 17:04:10.675\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m378\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", + "\u001b[32m2024-04-17 17:04:16.804\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m367\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" ] }, { @@ -309,15 +350,15 @@ " - What were the president's remarks regarding Ketanji Brown Jackson?\n", " - What statements has the president made about Ketanji Brown Jackson?\n", "\n", - "Multi Query Accuracy Score: 1.0\n" + "Multi Query Accuracy Score: 0.5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-04-10 14:09:29.142\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m376\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", - "\u001b[32m2024-04-10 14:09:53.095\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m365\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" + "\u001b[32m2024-04-17 17:04:22.027\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m378\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", + "\u001b[32m2024-04-17 17:04:44.033\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m367\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" ] }, { @@ -326,7 +367,7 @@ "text": [ "\n", "Question: What did the president say about Ketanji Brown Jackson\n", - "Response: The president mentioned that he had nominated Ketanji Brown Jackson to serve on the United States Supreme Court 4 days ago. He described her as one of the nation's top legal minds who will continue Justice Breyer’s legacy of excellence. He also mentioned that she is a former top litigator in private practice, a former federal public defender, and comes from a family of public school educators and police officers. Since her nomination, she has received a broad range of support, including from the Fraternal Order of Police and former judges appointed by both Democrats and Republicans.\n", + "Response: The president mentioned that he had nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court 4 days ago. He described her as one of the nation's top legal minds who will continue Justice Breyer’s legacy of excellence. He also mentioned that since her nomination, she has received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans.\n", "\n", "Context Relevance Score: 1.0\n", "Factual Accuracy Score: 1.0\n", @@ -374,9 +415,50 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-04-17 17:04:46.462\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m378\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", + "\u001b[32m2024-04-17 17:04:53.561\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m367\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Question: What did the president say about Ketanji Brown Jackson\n", + "\n", + "Context Conciseness Score: 0.0\n", + "Context Reranking Score: 1.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-04-17 17:04:56.947\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate_on_server\u001b[0m:\u001b[36m378\u001b[0m - \u001b[1mSending evaluation request for rows 0 to <50 to the Uptrain\u001b[0m\n", + "\u001b[32m2024-04-17 17:05:16.551\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36muptrain.framework.evalllm\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m367\u001b[0m - \u001b[1mLocal server not running, start the server to log data and visualize in the dashboard!\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Question: What did the president say about Ketanji Brown Jackson\n", + "Response: The President mentioned that he nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court 4 days ago. He described her as one of the nation's top legal minds who will continue Justice Breyer’s legacy of excellence.\n", + "\n", + "Context Relevance Score: 1.0\n", + "Factual Accuracy Score: 1.0\n", + "Response Completeness Score: 0.5\n" + ] + } + ], "source": [ "# Create the retriever\n", "compressor = FlashrankRerank()\n",