From db04580dfa69e182691a9e55f14e9c8e217ff159 Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:47:55 -0800 Subject: [PATCH] Add Gemini Notebook (#14661) --- .../chat/google_generative_ai.ipynb | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 docs/docs/integrations/chat/google_generative_ai.ipynb diff --git a/docs/docs/integrations/chat/google_generative_ai.ipynb b/docs/docs/integrations/chat/google_generative_ai.ipynb new file mode 100644 index 00000000000..d76315e3ff0 --- /dev/null +++ b/docs/docs/integrations/chat/google_generative_ai.ipynb @@ -0,0 +1,317 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "bf496c3b-3a09-4a59-ac9c-3c97153a3516", + "metadata": {}, + "source": [ + "---\n", + "sidebar_label: Google Generative AI\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "bb9e152f-a1dc-45df-a50c-60a8d7ecdf69", + "metadata": {}, + "source": [ + "# ChatGoogleGenerativeAI\n", + "\n", + "Access Google's `gemini` and `gemini-vision` models, as well as other generative models through `ChatGoogleGenerativeAI` class in the [langchain-google-genai](https://pypi.org/project/langchain-google-genai/) integration package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8757740-08a2-4833-8a68-c051dac506f4", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U --quiet langchain-google-genai pillow" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c0637bf4-10dc-4f6b-adc5-6b37c9b2a8b9", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "if \"GOOGLE_API_KEY\" not in os.environ:\n", + " os.environ[\"GOOGLE_API_KEY\"] = getpass(\"Provide your Google API Key\")" + ] + }, + { + "cell_type": "markdown", + "id": "0301b16d-3391-47ef-b024-a265c71e0dd6", + "metadata": {}, + "source": [ + "## Example usage" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6a22bc43-f640-4357-8dcf-fe20052b4f46", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_google_genai import ChatGoogleGenerativeAI" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bafde6be-a75d-443b-8981-4b7d3258a214", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "In realms where data streams like fervent tides,\n", + "Where algorithms dance and knowledge abides,\n", + "A tale unfolds of LangChain, grand and bold,\n", + "A ballad sung in bits and bytes untold.\n", + "\n", + "Amidst the codes and circuits' hum,\n", + "A spark ignited, a vision would come.\n", + "From minds of brilliance, a tapestry formed,\n", + "A model to learn, to comprehend, to transform.\n", + "\n", + "In layers deep, its architecture wove,\n", + "A neural network, ever-growing, in love.\n", + "With language's essence, it sought to entwine,\n", + "To unlock the secrets, each word's design.\n", + "\n", + "From texts vast and varied, it feasted and learned,\n", + "Its grasp on meaning, swiftly discerned.\n", + "Context and syntax, it embraced with grace,\n", + "Unraveling stories, at an astonishing pace.\n", + "\n", + "Translations sprang forth, with seamless art,\n", + "Bridging tongues and weaving hearts apart.\n", + "From English to French, Chinese to Spanish, behold,\n", + "LangChain's prowess, like a language untold.\n", + "\n", + "It summarized texts, with insights profound,\n", + "Extracting knowledge, without a sound.\n", + "Questions it answered, with eloquence rare,\n", + "A digital sage, beyond compare.\n", + "\n", + "Yet, its journey faced trials and tribulations,\n", + "Obstacles that tested its dedication.\n", + "Biases and errors, it sought to transcend,\n", + "For fairness and accuracy, it would contend.\n", + "\n", + "With every challenge, it emerged more strong,\n", + "Adaptive and resilient, all along.\n", + "For LangChain's purpose was etched in its core,\n", + "To empower humans, forevermore.\n", + "\n", + "In classrooms and workplaces, it lent its hand,\n", + "A tireless assistant, at every demand.\n", + "It aided students, in their quests for knowledge,\n", + "And professionals thrived, with its guidance and homage.\n", + "\n", + "As years unfurled, its fame grew wide,\n", + "A testament to its unwavering stride.\n", + "Researchers and scholars, they all took heed,\n", + "Of LangChain's brilliance, a groundbreaking deed.\n", + "\n", + "And so, the ballad of LangChain resounds,\n", + "A tribute to progress, where innovation abounds.\n", + "In the annals of AI, its name shall be etched,\n", + "A pioneer, forever in our hearts sketched.\n" + ] + } + ], + "source": [ + "llm = ChatGoogleGenerativeAI(model=\"gemini-pro\")\n", + "result = llm.invoke(\"Write a ballad about LangChain\")\n", + "print(result.content)" + ] + }, + { + "cell_type": "markdown", + "id": "40773fac-b24d-476d-91c8-2da8fed99b53", + "metadata": {}, + "source": [ + "## Streaming and Batching\n", + "\n", + "`ChatGoogleGenerativeAI` natively supports streaming and batching. Below is an example." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ce0bd90e-2afd-4189-a9d2-278c1f10ffd5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "There once was an AI named Bert,\n", + "Whose language skills were quite expert.\n", + "---\n", + "\n", + "With a vast dataset,\n", + "It could chat, even bet,\n", + "And write limericks, for what it's worth.\n", + "---\n" + ] + } + ], + "source": [ + "for chunk in llm.stream(\"Write a limerick about LLMs.\"):\n", + " print(chunk.content)\n", + " print(\"---\")\n", + "# Note that each chunk may contain more than one \"token\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a147766a-0051-4127-8db6-62c070dd7866", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n", + "8\n" + ] + } + ], + "source": [ + "results = llm.batch(\n", + " [\n", + " \"What's 2+2?\",\n", + " \"What's 3+5?\",\n", + " ]\n", + ")\n", + "for res in results:\n", + " print(res.content)" + ] + }, + { + "cell_type": "markdown", + "id": "5a6cca7d-9ff0-4a40-b45c-651ec8cc5012", + "metadata": {}, + "source": [ + "## Multimodal support\n", + "\n", + "To provide an image, pass a human message with contents of type `List[dict]`, where each dict contains either an image value (type of `image_url`) or a text (type of `text`) value.\n", + "The value of `image_url` can be any of the following:\n", + "\n", + "- A public image URL\n", + "- An accessible gcs file (e.g., \"gcs://path/to/file.png\")\n", + "- A local file path\n", + "- A base64 encoded image (e.g., `data:image/png;base64,abcd124`)\n", + "- A PIL image\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "06c59be4-c4c8-490c-b0c6-ebffa4e3a034", + "metadata": {}, + "outputs": [ + { + "data": { + "image/jpeg": "/9j/4QDeRXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwABAAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAABIAAAAAQAAAEgAAAABAAAABwAAkAcABAAAADAyMTABkQcABAAAAAECAwCGkgcAFgAAAMAAAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAACwBAAADoAQAAQAAACwBAAAAAAAAQVNDSUkAAABQaWNzdW0gSUQ6IDg2Nv/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/CABEIASwBLAMBIgACEQEDEQH/xAAbAAADAQEBAQEAAAAAAAAAAAAAAQIDBAUGB//EABgBAQEBAQEAAAAAAAAAAAAAAAABAgME/9oADAMBAAIQAxAAAAH7BKOfonKlqBpocs9WBlNqyNpEvTPTOqyuJVnqqwqixzqGWiksgDLSU5505t52nObHeasjO51hArPp8s583p0IdW5CpYRT1MX0VLzabTCy6IOeruuZdWRmUkiN86yWsWZyRZ0YvKyc6y3glq5JqaQCeuI8/od50uu2GhcURNyGt4ku6zUbvkK2iMq3vl0TXF6mVxnW8cs2dWXO7Iw7a1nz59DC5556eeyUzUlXKeqJef01UUVplR0PFmgVCNKlwW6XK2kcJ0VKSMpz1momdSqxWs3pgHWcjmnBncVm5sSa1lJo9Ys83qh0xFBLpi0hmt8wdBgS7GKubjKdRw5uZQtQQrBAkqlYk1Yk0IaqVSSRh7ha8vrzNGZO4BNVUWkzNGQU9ZmbVmK0zpS5uUmkSasQ1Yk1SVJEmhKlSTSJNH0q2nx+zKdJJnYMq2aZxUUs2aiJjWLnF1eYrFNpIWiqCiyFasgpWSUiVSSRqpVJmVSr6uWeL1wqiagGMkqFqHPPSq4o7Zs5b6OO51nN7yTpVmQ5skpmZvUcum1y8k9115x6Vp5t9nGZ4OevPMpM/UTb8fryNKWHpTHOul1yz1zLxnWl5se1S+Yeo7PLv0ivLXphx3rnTT2MjomzOLixKuexmfNvLw0neMy5szVpPrDOfL22M1Wxzh0LJlxM506xzXpyUq89czBwt5125rN4yg7eeNKrPS04726jz/ifvfncuL6H5Tp7cvo8nWmU7SYztKfRTyz5+/YcarpOXQ7DlpNZVyxG6XPHsUc2jg1fPVmudSDAdRVm5hpI4fmS+d5WS7c3zuOnPr+m+C+kl9mdFN5zrJ2FHLrKsM3VpktJIKViGgTQ3IrciaQhSpoE2SrEj5v3fz+59OrOvLn1BOW86T7PXz/S594Vo5e/8/u5/Qp+M9bO/dMN86EykMEmElBKtElBBaENgDR5beefJx6PLvlvLneMs9sRZnOnV9D8xtX1fZ8D72OngPN9eOjycu3o+S5frvW/PNMdP0E+U9HG/aPOqa7jl2NEMQwQ0oDRPVy5fMfQ/GXOV0duFxnnZuYKNInhO6cL1l83XBDkqiQsgNHm4tJmmnMpes46Xr14bl9Ts8By/V9vw9TX6BXwnoZ39dj4T5dMfFz8/tw7I4nvn6VKdTTh3Z5z7Msajp5dbnS8q3mEAAAAoAMRFOA0UMBskEU4DSsGvRfI17q8+o7K4XXZHOzcwE157JbgZnO5c84ixiBoAAUAgAAAYgbkKQCVBJSExDcMogLJCyAsgLIBjNRFEIoILCCwgsILFgoJKQhkSMEMAQMQCZYhqAEoAABsI1GSyiCLIC3mGhmLoZo1M2WQFEgwEStEqkSNAmCAAAQAgD//xAAoEAACAgEDAwUAAwEBAAAAAAAAAQIREgMQEwQgIQUiMDFAFCNBUHD/2gAIAQEAAQUC2e9dyZYr2ZQ47YtigNCdH2qPscShlmTG9vA+2xvZFjHvRQiyhiHtbFbMTHbItljaGvici97LL3rfES2a3orbyux7ZH2NIkX33tfekUu17KihrZIox2Y7KZ9LMcr+Vd99je2Re31smeColpDkiT+Zd1lmQ5GRZ5PPZ7a8DZZZZ9nHE44mCHGh/T+Nb1vRRRWzEMY5GRZZkWI8FolZ5+WyxMsSK2rtyHPw2WX22WWN/gW1ll72ZDmN/kortrtsssssf/drau6iu5/mooreijA8I+x9lll/rooorssssv8AU90xvsYyhIxK7KH3JjMTExOJnCNRiOu57VvR4PBcUZxPaScayMtvBa7KMWcbMGYMwOI4zBIclElqt/BQkUYlGJiYHGYpErMGzjOM4zjZxMWgjhiYJGK2psx2VsknSjM+iaK7aKKKKKKK7KKMTExPG3g8DmkchlYkUjIc0cqOWKJdTDKU3bbfw18WRkzNjlIzZysyRlEygZIzHJt/5kN21ptvU08IareXTdRiLFx/ztszL2dmTMjztkOzNGcTkQpJkpLdKzHarMWYyvBi02cXlJQWrP2ak4Ia9+l1D0XGSnDt8FozMkOQvO3kxKKkzjRgjAUWxacTGJ4PA1E9pkkJsoW1E5KC6nXzdHgbVdJ1PGffbkX2ZUcjOU5EZRMkZRMkWhyMzkOQzMhTHJnIcqFqGaM6Ou1I8ObtzY2SNJSR0XUW/wASo8FiY5X8Fs67WucYpJVdxlJ/SpSdp6Us9L4KPH5NfVWjpamrJy0743HxjWzrFe5dFqZaO8fU9KRDUhqfo1J8enqS1et1300IzG6E8iVsjCibo0tbGcfUdJmj1WnrvazT6zW0zT9SiyGvp6v5erip6c56UI+E9nJIep5m7J2VipJSXmD0vUVxllllml1utpml6hpTIyjNfh6ubeu1MgsUMqi1VE/CysypTmqxi+6yyOpKD0/UtSJD1GEj+bon8zpz+RosU4P4eOzjOonxdO9VSlk9nNIzSbkZo5lerOxMtjSkVJfFYptHImexmJ70LqNaJH1DXRH1NkfUdFkeo0Z7I5FAl1emj1DrMtBz8cpzkakNJnipqjxGTuUYwYn4+i/gsvtsyORmZki4ngjOcSPW68RdapLm0GdU3qSzolTIaeTXhWWWXFk5Us5aqSpeH8t7V2XvZZmzkYtZmaZ/Uf1n9ZhpmEDCJxpEtDKS06JabMJHn9VllllllllllmRl++y//Rf/xAAgEQADAAIDAAMBAQAAAAAAAAAAAREQEgIgMAMhMVBA/9oACAEDAQE/Aey8bh9540b/AMFGzVs1H6tlKUo2XM70pf4TH0nvBohCGpqamhoOIbxUU2NjY2KiouaUpTYf3mMhqQSILFPvpB8UfmZ2pfB8hieJ7cn0X5hfIxc0XzfRcmhc8rkL5DZFRe3P6WL5UpTY3FzRtxPkdzcP1pSl6T+1/8QAIBEAAwABBQEAAwAAAAAAAAAAAAEREgIQICEwQBNQYP/aAAgBAgEBPwHix+M2XOl8IL4IJFRfaE2hBInjCE+Nfo6UTEylMjIyMjMVYt4QhCEO9u+EEmYi680iIi2pkZOi4QhNpvOUol8SXB7fjHp9NK4QeneD0GLITjTR2+K5whDEwHoZizQpvCC9p/Af/8QAMRAAAgAEAwYEBgIDAAAAAAAAAAECESExEBJBICIwMkBRAzNhcRNCUIGRsSOAYGKC/9oACAEBAAY/AvqdCv1a30Kn+F3xv/VWxbprdJp0dOFYtsWlhVl+t128raTZp1Vdq+w3OxP4uZ+xJvd7diacyUugoV4Nim3MsmTRK8PYUUNuFcvw7FixY5TlOXZnE5GWFvJLY3uV9DfZtwb4ZPmboOUh1RWIZLQ+DF/z0dSmNuDJfLSY9TQlg4ZiadSGJ6rrs2uhFN1Fm7F3jm1KmR80P62N5OE3I4YvZ9RFG9EPL+7D3s0sJ4ZSo3DcUSo1ob8LgGob9nsc812Z/JBL1RuxrpYYIuRuo4PCVHzPvsXKQ4KVCeCaYsym/fa5sy/2N/cZOFpr06JpxfZDPUmySK4zFSh3wrwJwxNG+lEVha9mXa+x5iPNh/JSOH88OOJaD7li5I/RPQrYkiQkUwo+HcrDhRo1/J5kaOafuje8NfZlVFCU8RY8xWIywUm8JueE2VJEihNozfvo74WwubsbX3KtREovDl7HLX1wlPCtiS2JIkSSkir6ysMJ5aOU1NS7OYuTcZphYs/6F//EACcQAAMAAgICAQQDAQEBAAAAAAABESExEEFRYSAwQHGBkaGx0cHw/9oACAEBAAE/ISc0h+hB8NEJLg0YMzIz8Ga2QFdcNvfFBsT6MmOeQ38isUbF0F9oucEjLFVx8WacQRwj4kLYn6FCOCN6JCR3R5YGnZJplLBJb4sd5PePDK4VmEVQfxHXF5WGQXCMVECY41jZE1s28jYVJoY23sf4G8kPsk7PwEkhfoboYKQvyZvC8QRCcCCLxxBaFMLowaI6Qy6Glop2NfI8iDIRh5y7B+g76BspRMTIgxiExUTGNCY3BJVBZQjyOPRsJ+xHY6H4OM2dK4lIxprr434JjCfEpJxQo5FlMaDgSbYoRXY6D8EQ7H78SjeSJtQVMj2jR4OsTsxPopiZkMl4T4CChGfIwnoT0YbIDYYYnwOh0xNGNwyPBX24NN7Y55+dELhT8KQQRDQiQ9GTXFD4LLD5pgZGA1lG78oQgl8lEclvg37KYIaLgaN/J/XIQhBCGvmDcbhj+zhCE50U6KX4se+KN/ZzhODR38GEYuIThB/cRk4Zi8iBK9DTC5o+F0NjZfqz5sZCKc0SDRIeNcVDgYZQb4hCIaXxnE+m0SiTh+hi4Hkg0YLGSmPAqMYjg2b+JCEEJHLUFPoop9CYaD7Ko7PCnE+GZCiEIXBlhbcHyjYjgoyii0Q9A3eIV4E7o9J6BFYvIikORQMQmv0NtsfNL8Cw+WzRDy+F4BZmlifwL0HTBbZ3GQ9ngcSTXCYj9JsW4Xo8jnQhrvT9lXgYnD4vmQXwBrhrhmSfkQRCL0NLtml64LLoptfoYc4IDVYOwdJLaLoiFSYvGno2D4hCEIQhBhFRSlKNs32IaLQyYpd/wPzGGCZvRTICS6JZjLPDizaNF0mNwGRQxCwMzzMtN/AvZ9k2I0x6Kk6bdkGhohBuj8S2R9kHgXYj0MhuE9ndgQeREdccYuossqmlyr6aGphi2wEl4QmZFiHKhTTKIFZHp+gwwx6Qp6qe8HdA8t/4PyrZRCEIQbjI1EHsJaDt84+BRaQ60OtslaVY/ESF3D8I2uobQS6pLofu/RGUxOgU0XgTYYbyim24INPskhYnsezGTUQ6WG0Pav8AkqNZSHItrVLoURNOpq0hCEGzKKUouhI9AvQTdidpnvMwTexr7I6Sf7L9Gwx2iWjFUT2ISG3kwiW/Anho2RvMnQn47PWfnwPOEeO6Rm70tkxI3S2y8k6IoO+X+ExxCcz6mTIqO/2GzIbEBX5EZ56FUeRmL4nv2Ztmu2hYzbMg3CrWcC1kSzKbf+GIb/A7flt8TiPwQ7GxJGtJ4saXXM+neYQanj0S7Y7sx5/Ijd4JhJE8MhrR6laS2KzLY1kbQjlyOsqsnwfydtVH/rIDU3zX9GcQn0EJFVhxK2+1opsN6LyJvUwKRsJfV/RZolevBOLqH6Dq/wAsNH+S2hwTS/ma88Jow7IEb8iYT+Q/0bqPx3/H2iHCNUv6S/6Q7Py79Du584L1+ym6GiyeSpx4eKT2IvBfZezwvs1maHIasXAvIhif0ZyIiV/yI9gqb7CivQ4G0tMwPJEkm9eRib/wSuufWipoI2P9mZr8F1LDS8jJ7TGy9BzkpiSw0ILVUzYq+ilKUTFwfqyWYxD5eGNp+4BRz+e4nf8AJoWqLaD8FR6d+aTfQnS3+Rvna9i86adfkY0bbtoqlgjLz/Q3Gvv0NdnL7CXGaXkhKeiTRgwzsupYotwqa1noaHU10LF/NHiNS/BeKUpeMmeGmHQmXyL+z/4oLrn8DR/sM2KfvHbP6D/BCm4r8NxnVWV6HS2xoV0jZqOoi1n1kikTftCSuCYFnMeoYKL7J8aG9BSD0lSrDrwVZu2/JWYf9GsxM3/BSsKjb0WBY418LxS8EKiDTK0J0J120IXuk/0V2aeUf/LH9/GDFIfaKjJ7UeQmBY254WBZVlTQzJXP9LGSMK0InJuTdiqjkUplj/eImyVCw8mPp3haMRlLwpeCcSO2JfbEex76/KNrn9GS6fskRf2PE/5cB7gtL+glFH6F6MN6j/ZNbEX/AIfWpTBCcUpSlLyI154X8fsryX5/v7ClLxPhSlKUpeSlKUpfuIT6FKUpS8ziEIQhCEIT6VKX7OEIQhCEIQhCEIQhPssGClRSoqKiopSlRgwYIQhCfeX6b+t//9oADAMBAAIAAwAAABB4hz44McsfWByvubxlUx4T21GzG/mwPwppGB1SynzV8/FMvTNNE3qRY8gAyCTi8+x0q8PCGETdrXy1EC6pTKfmvgBr92sDCoXAJhHgxrT6IMMzU4AT7hQ6cthkJnMZLy2D6zHFmrOSLjhMB6C3ftmVqCf2R8kOl8+D5x4CiMiSjqdXPFqd7/WcZpO54K3DaN0lFXAJa+f90OsrnVwBVCspUV9ykChgOrie/a8t7ZAAjDCDThFLyOrJhwkoGe84pkEQEYu5krMyCz32s89r7s4Q8n0yBLdxPPLFd+ut/OiyoXiGF7c89vLPMN/df/eOt/c/PvXPPf8AxwNNDrzjCRjDjDBZDDjCHDDmjDPvTDDDp//EAB4RAAMBAAMBAQEBAAAAAAAAAAABERAgITFBMEBR/9oACAEDAQE/EGMSo0MXvExYxuPFWmLYiBJa8YxtjSdlPC8EUuzGVlKQfFDA5EnmzYJcKMhssajouoQmUuH0wxvGUvCZCEH0N430N49fG0hDwpTEeMg4OhwhNpRCHsHREbDYr4UyxMQQIVPMQNUPsIp+CpKmeDDDDQ6eDcBf3HXB9CFjalDb+lpGVSTsbXjgh0UYnCs18GzujbSo5vHzIQTHnZ2dlZ2VlykVBt72EZ4mMCR+fk0P8y55pf0pRi8PgxO0Xj5HjxBiZeC6Lxf+iZCdH1CBjnUrKQJ3Nyi4XKUpRWIOhRDjI4L+1/u/y//EAB8RAAMBAAIDAQEBAAAAAAAAAAABERAgITAxQUBRYf/aAAgBAgEBPxARYJi8IvYlesTSU4UelLiEJITXoRrkyY12QgkQglj+pIVid2lKXad4kxBBLWAoyFLwmiCWJCJk4UomUohEEIJCXhTKUQohNFQuiiYnifJ6i8AQQJPo0RI86GJGveSiCHccaUJsoVZBKiaYP9cRYx7GIGwXyEkiENEL1Cn3twgxGMQhRMjIyDZhaQiP2UpRMonToiKiIiEkdYjsokt6PGq9DUSe/F8Ge+DVkPRMav2fVDQUVxgrCD6KduyZCbBBH8IOx8g/uSiokOhx7zPHCIgSSIUabFUV8X53+R/i/8QAKBABAAICAQQCAgIDAQEAAAAAAQARITFBEFFhcYGRIKGx0TDB4UDw/9oACAEBAAE/EKuMGic9BYWOmbRamSYMwkQoxEBEHukXcg8VFs3ENLh4hBGRiUKLAxQGp5S+cF8Qjlupd1Qg1byRVbfhlrJ9yxdyRTmxHlkC4TqtEcmhALgBakF0EQjnHqG2IhcQZnmDvLA+FxRXpghaZVguVdjXuNdkWr5hjGINtDxBcuuWG3H1EDRKWWWIeML828xKrWg0tmfEmbRUEauyL3o6Cx2+I6jGBphKLbCjLwRFSKyWWBWYlNSzFR8lSdz+o3wiOLPqNdLHiNwO/MXoxxyAlXUcqw9zvyKmBpBhwijix7gLpy71Li67QVz4QHBEbrwjaXHUdR6XMfOEAYAkUQYjhDLcFMsEaH3CJM4IwDCD7JQZEDswKKhLDJJwiQVVscQXKIWLjRzS+SF3S9gmtMPClbrKTLjCGVmo5lSo6jqMqJe5eGeuhdDslBwS8tcWLugSzBuK7Zg9oOyRllFQvNxShK9wbCImvJiKN1ct7ukdwfUG2G+9x1giKKXCyHzFeZbSINVNoo66Jia3KR6EToYah0GAhMGIu0KtkBbtuK6Z3SxZYTURl2pogGxHzLitDwwbBEr4YP8ARKxLHmWTl8QbZ+EK4ycxItXxGmjLzNbFhYSpQ+SZsRJVyu8QuX0NQlXRZtMOZeFEAES+JSOTA7TIcwyuOFLlVVQhB3QGqvEtNGIsoKg8wV7ygpGO0vXAs/zM2AgCB9Swtr3Bv7Ucu+EOLRjFjGK94LvEzaCkqbj5wkzEKEQXED8JQLYY0dy0Wkw5Co0FLmdzcsN9NjNEvN9HYmTLBnDUQysXy2dNcXMdRjH8AVUubZUVMOUfV4lLJKNXcWK943BmYnERTF5i2xj10jqPRW9x3Hq6jqPTLpxhhNOkIioG0u4oQxnt+AHXLBsmkFxKY7jvo66OoxjuO5z1ddGMIeoqGcEcodBpqKWC3qZldLhOSXOXQWzpz0qOomOjEz05lSo66MZeK7SsBUHiasRLJWYDWor4nLEvcqAqNIltQTMpF3FilS+rKjqPRMxG5TfSmpUdR1HfQxSDgjclO0KvcHekrepSzUG+CPeVi5ToNFjrMAgRtw9HpUpjHXROiSsy3RjqMYlcQXxAzaUUhZ2QDgirtNsvPeOlLcU5Y0VO+l5ePuL3lhmdiUs84m9w5pT8yifEvxEJlEp4+4pdVEzKlZnxKYjecROLzEiRqBCyoXvFvmPum4Zbgl1FkW/HeIwLQJyd53UvZKIIgZ9QG0YwGYvYCBfFwo5zKcBKlS0VU9FjcQ+y5mr4UFbRO9Q1ui65K/EpCL3NT0TMtGyN1mXXuJVwbo6V5dxDIWCGnMqDzxFro9S8CzzACUnaPSh8RuUc3LXZBbsIRS6iQ0BZdslViy5d4X1BtI9zRTRmteYDwIzZAMBEcrED2IWgPll4Mu1y8CEi4KvuCI3MxlTN4i2YtSkwXPBL8XLdp45ku2XLZXwX8QUoAvEbyxniZ3Q+IpyYlBqDsoLfDhMGlWHIpQUVwrOiYrU1zFWEjxKYADt3A6vwS/e1ZzALmHC4FqLlTmGUQ7QFxIM5jU1KDqWyhmkC7lSUaJWclSxiF3UStzvVAnDodp9zmEeoBZ/M4leUxha+CF1SbSPCI4fkzZIlFjRPMpcqrxG4LviXLjHVl99HzKoQsjKWZe8Rdr9xtvMcIww5SvELzJmFlTtddyjArzLb3lADbn3MVtLrQweq+pkFsNWk+4DQnwTOuu0rC2ogatWNdK4lruamzp9RRXQ94ZQmUYVxUOCHm4K4KIIovmuIt2PRHyePEw/25C9y+IK8u6GyBhP2Ry6g9Fi4K6tHWEhsy+YrgK7XO1vUq1kvaDswSpsfbNwh9RI0O7KgsebiXCfBFUGOV3NPj2RnR2Kl2BXyEXHP1HWEgA3HmW7C+Zh1e9OoLEv3EJt2ZVtp8XMUA7sbkGco5lELeZRwLU2FRiWm0FPmIckaYxmhvEcvPY/zKEPsdnuMOEc9RzcRynkEp+8FJZoSzkP3LFdJBWCcRFyab7QLR8KGbVsANtYSwB6uMZ0eqmwT8ztX2RK0EeBBxEPiAwzw0pUoF0YtzAIzvEQVHljXCOwSkEoiXLHYlDYZjfT4rb2jfCwDnm7mcGrJ5hAqPaNEDN1qzzKZq1nXOf8AkRWEA0jpiKm8YcoQAs8syysaam1tz+toBsRQz+87J6Uo4bfMr0EDaOOYJrDNygwqA7XB73O4wGCVOMolEFJG2AuKKge5QqFuZ2Oe0yih3Y5VFhUVDzG6ArOrZ+FwoURZVq245kHZYd5Rx+ZR4QWgBReas2PEBTy9xuYYDZa/37kyse5UYc4FyplKzURvmUx7WdKzKe8bOZxiCnLFXlgGR8E3Q+ksLA9FS1gJDgAV5jMm9wMtvtgV4fEzd5uNtHiJLMPcjRSJnfqMBQxO8jfiVoUc4R4PEpiFMKsDz2mAPkQ78LGvAszevbDabIHtjDlZK5dyJcNRd0z/AKieIwmdRJyiY7lVaIlcROz9wLkxKbtPNyx2RMSo2iSpWZSMr/64nSuhL1UC5XUI5mWB8VyxU1JpvvzBhWqIeeYuqKhqYXdaot37hF0lkNfuY21ZLP1Ha9ADDfDFDygCnAprklYNRiSqF3GG/JmZm20B+jmKkwfMruJ7lSjSxy25Y/g3cpldFPQxTKYGJUMQLhmEsWZdXXEJExcAzRl1ELQouHkr7vEA2AO0QjAXAiNBlaEw67k8qQ76Xst2TKJIHCv9y/cA4vNPOOIIVWmWfWYnM8UnAH9S9Vn3iWLwPeFgisyNxenvB/cIEdy0gouXNr7JrdF660SiPWvxojAlHQgmeUhUZlX5BDIZx+r/AIK1C0URSCleiXcr1y1E2NY4YsVC8csQvG98H3FUFAVwQ19gzsPf8xsQjSox5lh5BYx8S77W7Df1r3N/XM75O/nmEFIY9ESsUe4wUPoT4dzyih/05glY4CH9fMdfg76uv8FJZe7xcFD1YweTvN6I40P6+IxjwBte3ctQHsk1dqpmUW0cq4+JZRkFFMXBmgVWLL9EbfBQXfqUqJ5LgliCnh8S7gWlyMAWpdjUOgYfgAyghUN3RKLnH+QmaE0gv3KJuOP5BcZoD3/qR6vnUf2Qf6Mi0no3KTY/g9EgE32h0UbgwmiC/wADwc8J7PMXq41BjjviYFUvFQIChvcPGlsq5QCSkrb7QQjaXMsTTlZ5lk6mHCvLDyjXe1wPVlbwSOWBcKXT2iXLoRlLYYfgLJWw/Eal5ljhlOKnZYOz3cwzgvjWhDkw7XGw6e9/cdMZ5v5JTg3YU/MPR3hb9NSqHnnB+v8AcrR5/wAAwL+QZEWXZjBLM+EC5ZAFaCNVVcimsi+IVl2uQWZVliXiNcUCzYfMoAvuMMoAGVIaiLblK8wYHIwmWb9ziFHbfeXwV8lKiy8Fw1WeEdaeB2grl5ZzLgm+GYNS5fRb1bVLcMsleHorG2dkmg9BgMWfeZd+2g4h5IIgW1n4YhNIcufpx+oZ3VCt+TMM8uNb6aYeKAqwgx4KW4QF4ajcECnfczMKFO8ZooVel8ShlFQRsQq6og3nXmNhRVcsow3K61MjqNo0/tiIoNFBv2s43dgVioNtflUWzs9S7l1Lly5cuXBlsvBygyQHTEJaWOYKXYGDNNQWR+ZoPum4+ZuYBCXdIT9JbKGF7otwr0R/EzAA43nIWKjH3xUaOzkGPLWu6JcWCDY/9zbpW9VUoSteikeCvCMcJb5wajn8zEuXL8y/M9oPljZtIlcRUtJaWgq30+0KS1VBnMTuy7lL955JaX8/ct5+4PUxVcC/pPK+35XLly5cuXLly5aWhwZREMSmI3My0tLwt0Ff8U/uXLly/wDFcuXBqCSyWTEqU6EagJ0WWw1LPwJD0SUdFf40AplP4rmWy2W79NJhlEQlE1+CY/ElSoa6GX4B/IFx10PUOX/kDDMuKdCnMLPwApx0U/AEUpxG7Meo66Mq5VR3+Drq9LlxcwcR3+Fy5cuXLXuXLly5cGXL6iJiOo9XfV11en//2Q==", + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import requests\n", + "from IPython.display import Image\n", + "\n", + "image_url = \"https://picsum.photos/seed/picsum/300/300\"\n", + "content = requests.get(image_url).content\n", + "Image(content)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "651d614e-1398-475d-9594-2eb441605d4d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content=' The image contains a snow-capped mountain peak.')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain_core.messages import HumanMessage\n", + "from langchain_google_genai import ChatGoogleGenerativeAI\n", + "\n", + "llm = ChatGoogleGenerativeAI(model=\"gemini-pro-vision\")\n", + "# example\n", + "message = HumanMessage(\n", + " content=[\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"What's in this image?\",\n", + " }, # You can optionally provide text parts\n", + " {\"type\": \"image_url\", \"image_url\": image_url},\n", + " ]\n", + ")\n", + "llm.invoke([message])" + ] + }, + { + "cell_type": "markdown", + "id": "eb2641f3-5dae-4756-b1c1-b58b41edc344", + "metadata": {}, + "source": [ + "## Gemini Prompting FAQs\n", + "\n", + "As of the time this doc was written (2024/12/12), Gemini has some restrictions on the types and structure of prompts it accepts. Specifically:\n", + "1. When providing multimodal (image) inputs, you are restricted to at most 1 message of \"human\" (user) type. You cannot pass multiple messages (though the single human message may have multiple content entries)\n", + "2. System messages are not accepted.\n", + "3. For regular chat conversations, messages must follow the human/ai/human/ai alternating pattern. You may not provide 2 AI or human messages in sequence.\n", + "4. Message may be blocked if they violate the safety checks of the LLM. In this case, the model will return an empty response." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}