From 344adad05681c2d79a1ff3ab3c7e35782ccda6ab Mon Sep 17 00:00:00 2001
From: Francesco Kruk <46400934+francesco-kruk@users.noreply.github.com>
Date: Fri, 7 Jun 2024 02:02:20 +0200
Subject: [PATCH] docs: Update jina embedding notebook to include multimodal
 capability (#22594)

After merging the [PR #22416 to include Jina AI multimodal
capabilities](https://github.com/langchain-ai/langchain/pull/22416), we
updated the Jina AI embedding notebook accordingly.
---
 .../integrations/text_embedding/jina.ipynb    | 108 ++++++++++++++++--
 1 file changed, 101 insertions(+), 7 deletions(-)

diff --git a/docs/docs/integrations/text_embedding/jina.ipynb b/docs/docs/integrations/text_embedding/jina.ipynb
index 111181ad45d..b211ecff101 100644
--- a/docs/docs/integrations/text_embedding/jina.ipynb
+++ b/docs/docs/integrations/text_embedding/jina.ipynb
@@ -10,6 +10,16 @@
     "Let's load the Jina Embedding class."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f0a1567-6273-47a3-b19d-c30af2470810",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install -U langchain-community"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -17,7 +27,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_community.embeddings import JinaEmbeddings"
+    "import requests\n",
+    "from langchain_community.embeddings import JinaEmbeddings\n",
+    "from numpy import dot\n",
+    "from numpy.linalg import norm\n",
+    "from PIL import Image"
    ]
   },
   {
@@ -27,9 +41,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "embeddings = JinaEmbeddings(\n",
+    "text_embeddings = JinaEmbeddings(\n",
     "    jina_api_key=\"jina_*\", model_name=\"jina-embeddings-v2-base-en\"\n",
-    ")"
+    ")\n",
+    "\n",
+    "image_embeddings = JinaEmbeddings(jina_api_key=\"jina_*\", model_name=\"jina-clip-v1\")"
    ]
   },
   {
@@ -39,7 +55,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "text = \"This is a test document.\""
+    "text = \"This is a test document.\"\n",
+    "\n",
+    "image = \"https://avatars.githubusercontent.com/u/126733545?v=4\"\n",
+    "\n",
+    "description = \"Logo of a parrot and a chain on green background\"\n",
+    "\n",
+    "im = Image.open(requests.get(image, stream=True).raw)\n",
+    "print(\"Image:\")\n",
+    "display(im)"
    ]
   },
   {
@@ -49,7 +73,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "query_result = embeddings.embed_query(text)"
+    "query_result = text_embeddings.embed_query(text)"
    ]
   },
   {
@@ -69,7 +93,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "doc_result = embeddings.embed_documents([text])"
+    "doc_result = text_embeddings.embed_documents([text])"
    ]
   },
   {
@@ -81,6 +105,76 @@
    "source": [
     "print(doc_result)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac2aace1-27af-4c4f-96f8-8e8b20d95b98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_result = image_embeddings.embed_images([image])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6687808c-1977-4128-a960-888bb82c46e1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(image_result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2844ef7c-cf9b-4e28-b627-09887aaa0a6d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "description_result = image_embeddings.embed_documents([description])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23372332-2ea3-4e4a-abc8-8307d45ebc95",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(description_result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "08d3ba5e-8957-4b10-97e3-40359ab165a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cosine_similarity = dot(image_result[0], description_result[0]) / (\n",
+    "    norm(image_result[0]) * norm(description_result[0])\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6be56ff9-774b-4347-a5cf-57d8db9e2cf2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(cosine_similarity)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f280807-a02b-4d4e-8ebd-01be33117999",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -99,7 +193,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.11"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,