From 7f0cb3bfba469965cfa5618834184e95e35c2e08 Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Thu, 4 Apr 2024 12:49:15 -0700 Subject: [PATCH] docs[patch]: Make Docusaurus and Vercel add trailing slashes when navigating by default (#20014) Should hopefully avoid weird broken link edge cases. Relative links now trip up the Docusaurus broken link checker, so this PR also removes them. Also snuck in a small addition about asyncio --- docs/docs/contributing/code.mdx | 2 +- docs/docs/contributing/integrations.mdx | 4 +- docs/docs/expression_language/interface.ipynb | 2 +- .../integrations/adapters/openai-old.ipynb | 2 +- docs/docs/integrations/adapters/openai.ipynb | 2 +- docs/docs/integrations/chat/ernie.ipynb | 2 +- .../providers/mlflow_ai_gateway.mdx | 2 +- .../integrations/stores/upstash_redis.ipynb | 2 +- .../integrations/text_embedding/ernie.ipynb | 2 +- .../text_embedding/nvidia_ai_endpoints.ipynb | 2 +- .../agent_types/openai_functions_agent.ipynb | 2 +- docs/docs/modules/agents/quick_start.ipynb | 16 ++-- .../model_io/chat/function_calling.mdx | 14 +-- docs/docs/modules/model_io/concepts.mdx | 6 +- docs/docs/modules/model_io/index.mdx | 1 - .../prompts/few_shot_examples_chat.ipynb | 2 +- .../model_io/prompts/quick_start.ipynb | 4 +- docs/docs/modules/model_io/quick_start.mdx | 14 ++- .../question_answering/quickstart.mdx | 90 +++++++++---------- docs/docusaurus.config.js | 2 +- docs/vercel.json | 1 + 21 files changed, 84 insertions(+), 90 deletions(-) diff --git a/docs/docs/contributing/code.mdx b/docs/docs/contributing/code.mdx index d3f957d1f89..78258317635 100644 --- a/docs/docs/contributing/code.mdx +++ b/docs/docs/contributing/code.mdx @@ -98,7 +98,7 @@ To run unit tests in Docker: make docker_tests ``` -There are also [integration tests and code-coverage](./testing) available. +There are also [integration tests and code-coverage](/docs/contributing/testing/) available. ### Only develop langchain_core or langchain_experimental diff --git a/docs/docs/contributing/integrations.mdx b/docs/docs/contributing/integrations.mdx index 7c01c95f96d..6f20b596b33 100644 --- a/docs/docs/contributing/integrations.mdx +++ b/docs/docs/contributing/integrations.mdx @@ -3,7 +3,7 @@ sidebar_position: 5 --- # Contribute Integrations -To begin, make sure you have all the dependencies outlined in guide on [Contributing Code](./code). +To begin, make sure you have all the dependencies outlined in guide on [Contributing Code](/docs/contributing/code/). There are a few different places you can contribute integrations for LangChain: @@ -133,7 +133,7 @@ By default, this will include stubs for a Chat Model, an LLM, and/or a Vector St Some basic tests are presented in the `tests/` directory. You should add more tests to cover your package's functionality. -For information on running and implementing tests, see the [Testing guide](./testing). +For information on running and implementing tests, see the [Testing guide](/docs/contributing/testing/). ### Write documentation diff --git a/docs/docs/expression_language/interface.ipynb b/docs/docs/expression_language/interface.ipynb index 8cb8c8b8a7c..88485abd50e 100644 --- a/docs/docs/expression_language/interface.ipynb +++ b/docs/docs/expression_language/interface.ipynb @@ -25,7 +25,7 @@ "- [`invoke`](#invoke): call the chain on an input\n", "- [`batch`](#batch): call the chain on a list of inputs\n", "\n", - "These also have corresponding async methods:\n", + "These also have corresponding async methods that should be used with [asyncio](https://docs.python.org/3/library/asyncio.html) `await` syntax for concurrency:\n", "\n", "- [`astream`](#async-stream): stream back chunks of the response async\n", "- [`ainvoke`](#async-invoke): call the chain on an input async\n", diff --git a/docs/docs/integrations/adapters/openai-old.ipynb b/docs/docs/integrations/adapters/openai-old.ipynb index 08dfbc6c729..85add308454 100644 --- a/docs/docs/integrations/adapters/openai-old.ipynb +++ b/docs/docs/integrations/adapters/openai-old.ipynb @@ -7,7 +7,7 @@ "source": [ "# OpenAI Adapter(Old)\n", "\n", - "**Please ensure OpenAI library is less than 1.0.0; otherwise, refer to the newer doc [OpenAI Adapter](./openai).**\n", + "**Please ensure OpenAI library is less than 1.0.0; otherwise, refer to the newer doc [OpenAI Adapter](/docs/integrations/adapters/openai/).**\n", "\n", "A lot of people get started with OpenAI but want to explore other models. LangChain's integrations with many model providers make this easy to do so. While LangChain has it's own message and model APIs, we've also made it as easy as possible to explore other models by exposing an adapter to adapt LangChain models to the OpenAI api.\n", "\n", diff --git a/docs/docs/integrations/adapters/openai.ipynb b/docs/docs/integrations/adapters/openai.ipynb index d14f9809944..05fe49fda6c 100644 --- a/docs/docs/integrations/adapters/openai.ipynb +++ b/docs/docs/integrations/adapters/openai.ipynb @@ -7,7 +7,7 @@ "source": [ "# OpenAI Adapter\n", "\n", - "**Please ensure OpenAI library is version 1.0.0 or higher; otherwise, refer to the older doc [OpenAI Adapter(Old)](./openai-old).**\n", + "**Please ensure OpenAI library is version 1.0.0 or higher; otherwise, refer to the older doc [OpenAI Adapter(Old)](/docs/integrations/adapters/openai-old/).**\n", "\n", "A lot of people get started with OpenAI but want to explore other models. LangChain's integrations with many model providers make this easy to do so. While LangChain has it's own message and model APIs, we've also made it as easy as possible to explore other models by exposing an adapter to adapt LangChain models to the OpenAI api.\n", "\n", diff --git a/docs/docs/integrations/chat/ernie.ipynb b/docs/docs/integrations/chat/ernie.ipynb index cd0e121b06b..f5d2fa0e30c 100644 --- a/docs/docs/integrations/chat/ernie.ipynb +++ b/docs/docs/integrations/chat/ernie.ipynb @@ -28,7 +28,7 @@ "We recommend users using `langchain_community.chat_models.ErnieBotChat` \n", "to use `langchain_community.chat_models.QianfanChatEndpoint` instead.\n", "\n", - "documentation for `QianfanChatEndpoint` is [here](./baidu_qianfan_endpoint).\n", + "documentation for `QianfanChatEndpoint` is [here](/docs/integrations/chat/baidu_qianfan_endpoint/).\n", "\n", "they are 4 why we recommend users to use `QianfanChatEndpoint`:\n", "\n", diff --git a/docs/docs/integrations/providers/mlflow_ai_gateway.mdx b/docs/docs/integrations/providers/mlflow_ai_gateway.mdx index fbe59c84650..e1fa804f032 100644 --- a/docs/docs/integrations/providers/mlflow_ai_gateway.mdx +++ b/docs/docs/integrations/providers/mlflow_ai_gateway.mdx @@ -2,7 +2,7 @@ :::warning -MLflow AI Gateway has been deprecated. Please use [MLflow Deployments for LLMs](./mlflow) instead. +MLflow AI Gateway has been deprecated. Please use [MLflow Deployments for LLMs](/docs/integrations/providers/mlflow/) instead. ::: diff --git a/docs/docs/integrations/stores/upstash_redis.ipynb b/docs/docs/integrations/stores/upstash_redis.ipynb index f32a510b6cd..186dbf93cab 100644 --- a/docs/docs/integrations/stores/upstash_redis.ipynb +++ b/docs/docs/integrations/stores/upstash_redis.ipynb @@ -17,7 +17,7 @@ "\n", "The `UpstashRedisStore` is an implementation of `ByteStore` that stores everything in your Upstash-hosted Redis instance.\n", "\n", - "To use the base `RedisStore` instead, see [this guide](./redis)\n", + "To use the base `RedisStore` instead, see [this guide](/docs/integrations/stores/redis/)\n", "\n", "To configure Upstash Redis, follow our [Upstash guide](/docs/integrations/providers/upstash)." ] diff --git a/docs/docs/integrations/text_embedding/ernie.ipynb b/docs/docs/integrations/text_embedding/ernie.ipynb index d04bc4b9602..7abd2122669 100644 --- a/docs/docs/integrations/text_embedding/ernie.ipynb +++ b/docs/docs/integrations/text_embedding/ernie.ipynb @@ -19,7 +19,7 @@ "We recommend users using `langchain_community.embeddings.ErnieEmbeddings` \n", "to use `langchain_community.embeddings.QianfanEmbeddingsEndpoint` instead.\n", "\n", - "documentation for `QianfanEmbeddingsEndpoint` is [here](./baidu_qianfan_endpoint).\n", + "documentation for `QianfanEmbeddingsEndpoint` is [here](/docs/integrations/text_embedding/baidu_qianfan_endpoint/).\n", "\n", "they are 2 why we recommend users to use `QianfanEmbeddingsEndpoint`:\n", "\n", diff --git a/docs/docs/integrations/text_embedding/nvidia_ai_endpoints.ipynb b/docs/docs/integrations/text_embedding/nvidia_ai_endpoints.ipynb index ce23d469c40..6490dcb8697 100644 --- a/docs/docs/integrations/text_embedding/nvidia_ai_endpoints.ipynb +++ b/docs/docs/integrations/text_embedding/nvidia_ai_endpoints.ipynb @@ -16,7 +16,7 @@ "\n", "This example goes over how to use LangChain to interact with the supported [NVIDIA Retrieval QA Embedding Model](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-foundation/models/nvolve-40k) for [retrieval-augmented generation](https://developer.nvidia.com/blog/build-enterprise-retrieval-augmented-generation-apps-with-nvidia-retrieval-qa-embedding-model/) via the `NVIDIAEmbeddings` class.\n", "\n", - "For more information on accessing the chat models through this api, check out the [ChatNVIDIA](../chat/nvidia_ai_endpoints) documentation." + "For more information on accessing the chat models through this api, check out the [ChatNVIDIA](/docs/integrations/chat/nvidia_ai_endpoints/) documentation." ] }, { diff --git a/docs/docs/modules/agents/agent_types/openai_functions_agent.ipynb b/docs/docs/modules/agents/agent_types/openai_functions_agent.ipynb index 59b48600447..7e0b5e8644d 100644 --- a/docs/docs/modules/agents/agent_types/openai_functions_agent.ipynb +++ b/docs/docs/modules/agents/agent_types/openai_functions_agent.ipynb @@ -23,7 +23,7 @@ "\n", "See the following links for more information:\n", "\n", - "[OpenAI Tools](./openai_tools)\n", + "[OpenAI Tools](/docs/modules/agents/agent_types/openai_tools/)\n", "\n", "[OpenAI chat create](https://platform.openai.com/docs/api-reference/chat/create)\n", "\n", diff --git a/docs/docs/modules/agents/quick_start.ipynb b/docs/docs/modules/agents/quick_start.ipynb index 5b7e4ee517b..ee4d1ea12d6 100644 --- a/docs/docs/modules/agents/quick_start.ipynb +++ b/docs/docs/modules/agents/quick_start.ipynb @@ -20,7 +20,7 @@ "\n", "To best understand the agent framework, let's build an agent that has two tools: one to look things up online, and one to look up specific data that we've loaded into a index.\n", "\n", - "This will assume knowledge of [LLMs](../model_io) and [retrieval](../data_connection) so if you haven't already explored those sections, it is recommended you do so.\n", + "This will assume knowledge of [LLMs](/docs/modules/model_io/) and [retrieval](/docs/modules/data_connection/) so if you haven't already explored those sections, it is recommended you do so.\n", "\n", "## Setup: LangSmith\n", "\n", @@ -107,7 +107,7 @@ "source": [ "### Retriever\n", "\n", - "We will also create a retriever over some data of our own. For a deeper explanation of each step here, see [this section](/docs/modules/data_connection/)" + "We will also create a retriever over some data of our own. For a deeper explanation of each step here, see [this section](/docs/modules/data_connection/)." ] }, { @@ -211,7 +211,7 @@ "source": [ "## Create the agent\n", "\n", - "Now that we have defined the tools, we can create the agent. We will be using an OpenAI Functions agent - for more information on this type of agent, as well as other options, see [this guide](./agent_types)\n", + "Now that we have defined the tools, we can create the agent. We will be using an OpenAI Functions agent - for more information on this type of agent, as well as other options, see [this guide](/docs/modules/agents/agent_types/).\n", "\n", "First, we choose the LLM we want to be guiding the agent." ] @@ -273,7 +273,7 @@ "id": "f8014c9d", "metadata": {}, "source": [ - "Now, we can initalize the agent with the LLM, the prompt, and the tools. The agent is responsible for taking in input and deciding what actions to take. Crucially, the Agent does not execute those actions - that is done by the AgentExecutor (next step). For more information about how to think about these components, see our [conceptual guide](./concepts)" + "Now, we can initalize the agent with the LLM, the prompt, and the tools. The agent is responsible for taking in input and deciding what actions to take. Crucially, the Agent does not execute those actions - that is done by the AgentExecutor (next step). For more information about how to think about these components, see our [conceptual guide](/docs/modules/agents/concepts/)." ] }, { @@ -293,7 +293,7 @@ "id": "1a58c9f8", "metadata": {}, "source": [ - "Finally, we combine the agent (the brains) with the tools inside the AgentExecutor (which will repeatedly call the agent and execute tools). For more information about how to think about these components, see our [conceptual guide](./concepts)" + "Finally, we combine the agent (the brains) with the tools inside the AgentExecutor (which will repeatedly call the agent and execute tools)." ] }, { @@ -544,7 +544,7 @@ "id": "07b3bcf2", "metadata": {}, "source": [ - "If we want to keep track of these messages automatically, we can wrap this in a RunnableWithMessageHistory. For more information on how to use this, see [this guide](/docs/expression_language/how_to/message_history)" + "If we want to keep track of these messages automatically, we can wrap this in a RunnableWithMessageHistory. For more information on how to use this, see [this guide](/docs/expression_language/how_to/message_history/)." ] }, { @@ -673,7 +673,7 @@ "source": [ "## Conclusion\n", "\n", - "That's a wrap! In this quick start we covered how to create a simple agent. Agents are a complex topic, and there's lot to learn! Head back to the [main agent page](./) to find more resources on conceptual guides, different types of agents, how to create custom tools, and more!" + "That's a wrap! In this quick start we covered how to create a simple agent. Agents are a complex topic, and there's lot to learn! Head back to the [main agent page](/docs/modules/agents/) to find more resources on conceptual guides, different types of agents, how to create custom tools, and more!" ] } ], @@ -693,7 +693,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.10.5" } }, "nbformat": 4, diff --git a/docs/docs/modules/model_io/chat/function_calling.mdx b/docs/docs/modules/model_io/chat/function_calling.mdx index d06c52bf118..148a67e51e3 100644 --- a/docs/docs/modules/model_io/chat/function_calling.mdx +++ b/docs/docs/modules/model_io/chat/function_calling.mdx @@ -12,7 +12,7 @@ etc., have a function-calling API that lets you describe functions and their arguments, and have the model return a JSON object with a function to invoke and the inputs to that function. Function-calling is extremely useful for building [tool-using chains and -agents](../../../../docs/use_cases/tool_use/), and for getting +agents](/docs/use_cases/tool_use/), and for getting structured outputs from models more generally. LangChain comes with a number of utilities to make function-calling @@ -28,9 +28,9 @@ easy. Namely, it comes with: We’ll focus here on the first two points. For a detailed guide on output parsing check out the [OpenAI Tools output -parsers](../../../../docs/modules/model_io/output_parsers/types/openai_tools) +parsers](/docs/modules/model_io/output_parsers/types/openai_tools/) and to see the structured output chains check out the [Structured output -guide](/docs/modules/model_io/chat/structured_output). +guide](/docs/modules/model_io/chat/structured_output/). Before getting started make sure you have `langchain-core` installed. @@ -298,13 +298,13 @@ print(json.dumps(convert_to_openai_tool(Multiply()), indent=2)) ## Next steps - **Output parsing**: See [OpenAI Tools output - parsers](../../../../docs/modules/model_io/output_parsers/types/openai_tools) + parsers](/docs/modules/model_io/output_parsers/types/openai_tools/) and [OpenAI Functions output - parsers](../../../../docs/modules/model_io/output_parsers/types/openai_functions) + parsers](/docs/modules/model_io/output_parsers/types/openai_functions/) to learn about extracting the function calling API responses into various formats. -- **Structured output chains**: [Some models have constructors](/docs/modules/model_io/chat/structured_output) that +- **Structured output chains**: [Some models have constructors](/docs/modules/model_io/chat/structured_output/) that handle creating a structured output chain for you. - **Tool use**: See how to construct chains and agents that actually call the invoked tools in [these - guides](../../../../docs/use_cases/tool_use/). + guides](/docs/use_cases/tool_use/). diff --git a/docs/docs/modules/model_io/concepts.mdx b/docs/docs/modules/model_io/concepts.mdx index 0074ce24b49..1db2cc6a0fc 100644 --- a/docs/docs/modules/model_io/concepts.mdx +++ b/docs/docs/modules/model_io/concepts.mdx @@ -109,8 +109,4 @@ There are a few parsers dedicated to working with OpenAI function calling. They ### Agent Output Parsers -[Agents](../agents) are systems that use language models to determine what steps to take. The output of a language model therefore needs to be parsed into some schema that can represent what actions (if any) are to be taken. AgentOutputParsers are responsible for taking raw LLM or ChatModel output and converting it to that schema. The logic inside these output parsers can differ depending on the model and prompting strategy being used. - - - - +[Agents](/docs/modules/agents/) are systems that use language models to determine what steps to take. The output of a language model therefore needs to be parsed into some schema that can represent what actions (if any) are to be taken. AgentOutputParsers are responsible for taking raw LLM or ChatModel output and converting it to that schema. The logic inside these output parsers can differ depending on the model and prompting strategy being used. diff --git a/docs/docs/modules/model_io/index.mdx b/docs/docs/modules/model_io/index.mdx index 6ffacb4aff7..614e0d6e7b5 100644 --- a/docs/docs/modules/model_io/index.mdx +++ b/docs/docs/modules/model_io/index.mdx @@ -157,7 +157,6 @@ You can initialize them with parameters like `temperature` and others, and pass The main difference between them is their input and output schemas. The LLM objects take string as input and output string. The ChatModel objects take a list of messages as input and output a message. -For a deeper conceptual explanation of this difference please see [this documentation](./concepts) We can see the difference between an LLM and a ChatModel when we invoke it. diff --git a/docs/docs/modules/model_io/prompts/few_shot_examples_chat.ipynb b/docs/docs/modules/model_io/prompts/few_shot_examples_chat.ipynb index 526bff05f0b..5cb2e99c91f 100644 --- a/docs/docs/modules/model_io/prompts/few_shot_examples_chat.ipynb +++ b/docs/docs/modules/model_io/prompts/few_shot_examples_chat.ipynb @@ -22,7 +22,7 @@ "The goal of few-shot prompt templates are to dynamically select examples based on an input, and then format the examples in a final prompt to provide for the model.\n", "\n", "\n", - "**Note:** The following code examples are for chat models. For similar few-shot prompt examples for completion models (LLMs), see the [few-shot prompt templates](few_shot_examples) guide." + "**Note:** The following code examples are for chat models. For similar few-shot prompt examples for completion models (LLMs), see the [few-shot prompt templates](/docs/modules/model_io/prompts/few_shot_examples/) guide." ] }, { diff --git a/docs/docs/modules/model_io/prompts/quick_start.ipynb b/docs/docs/modules/model_io/prompts/quick_start.ipynb index 9eabff4d368..bad77c66cb9 100644 --- a/docs/docs/modules/model_io/prompts/quick_start.ipynb +++ b/docs/docs/modules/model_io/prompts/quick_start.ipynb @@ -102,11 +102,11 @@ "metadata": {}, "source": [ "You can create custom prompt templates that format the prompt in any way you want.\n", - "For more information, see [Prompt Template Composition](./composition).\n", + "For more information, see [Prompt Template Composition](/docs/modules/model_io/prompts/composition/).\n", "\n", "## `ChatPromptTemplate`\n", "\n", - "The prompt to [chat models](../chat) is a list of [chat messages](/docs/modules/model_io/chat/message_types).\n", + "The prompt to [chat models](/docs/modules/model_io/chat)/ is a list of [chat messages](/docs/modules/model_io/chat/message_types/).\n", "\n", "Each chat message is associated with content, and an additional parameter called `role`.\n", "For example, in the OpenAI [Chat Completions API](https://platform.openai.com/docs/guides/chat/introduction), a chat message can be associated with an AI assistant, a human or a system role.\n", diff --git a/docs/docs/modules/model_io/quick_start.mdx b/docs/docs/modules/model_io/quick_start.mdx index 0b782cb8d84..f568efbd409 100644 --- a/docs/docs/modules/model_io/quick_start.mdx +++ b/docs/docs/modules/model_io/quick_start.mdx @@ -4,7 +4,7 @@ sidebar_position: 0 # Quickstart -The quick start will cover the basics of working with language models. It will introduce the two different types of models - LLMs and ChatModels. It will then cover how to use PromptTemplates to format the inputs to these models, and how to use Output Parsers to work with the outputs. For a deeper conceptual guide into these topics - please see [this documentation](./concepts) +The quick start will cover the basics of working with language models. It will introduce the two different types of models - LLMs and ChatModels. It will then cover how to use PromptTemplates to format the inputs to these models, and how to use Output Parsers to work with the outputs. ## Models For this getting started guide, we will provide a few options: using an API like Anthropic or OpenAI, or using a local open source model via Ollama. @@ -132,7 +132,6 @@ You can initialize them with parameters like `temperature` and others, and pass The main difference between them is their input and output schemas. The LLM objects take string as input and output string. The ChatModel objects take a list of messages as input and output a message. -For a deeper conceptual explanation of this difference please see [this documentation](./concepts) We can see the difference between an LLM and a ChatModel when we invoke it. @@ -247,14 +246,13 @@ chain.invoke({"text": "colors"}) Note that we are using the `|` syntax to join these components together. This `|` syntax is powered by the LangChain Expression Language (LCEL) and relies on the universal `Runnable` interface that all of these objects implement. -To learn more about LCEL, read the documentation [here](/docs/expression_language). +To learn more about LCEL, read the documentation [here](/docs/expression_language/). ## Conclusion That's it for getting started with prompts, models, and output parsers! This just covered the surface of what there is to learn. For more information, check out: -- The [conceptual guide](./concepts) for information about the concepts presented here -- The [prompt section](./prompts) for information on how to work with prompt templates -- The [LLM section](./llms) for more information on the LLM interface -- The [ChatModel section](./chat) for more information on the ChatModel interface -- The [output parser section](./output_parsers) for information about the different types of output parsers. \ No newline at end of file +- The [prompt section](/docs/modules/model_io/prompts/) for information on how to work with prompt templates +- The [LLM section](/docs/modules/model_io/llms/) for more information on the LLM interface +- The [ChatModel section](/docs/modules/model_io/chat/) for more information on the ChatModel interface +- The [output parser section](/docs/modules/model_io/output_parsers/) for information about the different types of output parsers. \ No newline at end of file diff --git a/docs/docs/use_cases/question_answering/quickstart.mdx b/docs/docs/use_cases/question_answering/quickstart.mdx index 9ccce7eaaef..8ec6ac9068c 100644 --- a/docs/docs/use_cases/question_answering/quickstart.mdx +++ b/docs/docs/use_cases/question_answering/quickstart.mdx @@ -18,7 +18,7 @@ complexity. ## Architecture We’ll create a typical RAG application as outlined in the [Q&A -introduction](../../../docs/use_cases/question_answering/), which has +introduction](/docs/use_cases/question_answering/), which has two main components: **Indexing**: a pipeline for ingesting data from a source and indexing @@ -33,28 +33,28 @@ The full sequence from raw data to answer will look like: ### Indexing 1. **Load**: First we need to load our data. We’ll use - [DocumentLoaders](../../../docs/modules/data_connection/document_loaders/) + [DocumentLoaders](/docs/modules/data_connection/document_loaders/) for this. 2. **Split**: [Text - splitters](../../../docs/modules/data_connection/document_transformers/) + splitters](/docs/modules/data_connection/document_transformers/) break large `Documents` into smaller chunks. This is useful both for indexing data and for passing it in to a model, since large chunks are harder to search over and won’t fit in a model’s finite context window. 3. **Store**: We need somewhere to store and index our splits, so that they can later be searched over. This is often done using a - [VectorStore](../../../docs/modules/data_connection/vectorstores/) + [VectorStore](/docs/modules/data_connection/vectorstores/) and - [Embeddings](../../../docs/modules/data_connection/text_embedding/) + [Embeddings](/docs/modules/data_connection/text_embedding/) model. ### Retrieval and generation 1. **Retrieve**: Given a user input, relevant splits are retrieved from storage using a - [Retriever](../../../docs/modules/data_connection/retrievers/). -2. **Generate**: A [ChatModel](../../../docs/modules/model_io/chat/) / - [LLM](../../../docs/modules/model_io/llms/) produces an answer using + [Retriever](/docs/modules/data_connection/retrievers/). +2. **Generate**: A [ChatModel](/docs/modules/model_io/chat/) / + [LLM](/docs/modules/model_io/llms/) produces an answer using a prompt that includes the question and the retrieved data ## Setup @@ -63,11 +63,11 @@ The full sequence from raw data to answer will look like: We’ll use an OpenAI chat model and embeddings and a Chroma vector store in this walkthrough, but everything shown here works with any -[ChatModel](../../../docs/modules/model_io/chat/) or -[LLM](../../../docs/modules/model_io/llms/), -[Embeddings](../../../docs/modules/data_connection/text_embedding/), and -[VectorStore](../../../docs/modules/data_connection/vectorstores/) or -[Retriever](../../../docs/modules/data_connection/retrievers/). +[ChatModel](/docs/modules/model_io/chat/) or +[LLM](/docs/modules/model_io/llms/), +[Embeddings](/docs/modules/data_connection/text_embedding/), and +[VectorStore](/docs/modules/data_connection/vectorstores/) or +[Retriever](/docs/modules/data_connection/retrievers/). We’ll use the following packages: @@ -187,7 +187,7 @@ going on. ## 1. Indexing: Load {#indexing-load} We need to first load the blog post contents. We can use -[DocumentLoaders](../../../docs/modules/data_connection/document_loaders/) +[DocumentLoaders](/docs/modules/data_connection/document_loaders/) for this, which are objects that load in data from a source and return a list of [Documents](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.base.Document.html). @@ -195,7 +195,7 @@ A `Document` is an object with some `page_content` (str) and `metadata` (dict). In this case we’ll use the -[WebBaseLoader](../../../docs/integrations/document_loaders/web_base), +[WebBaseLoader](/docs/integrations/document_loaders/web_base), which uses `urllib` to load HTML from web URLs and `BeautifulSoup` to parse it to text. We can customize the HTML -\> text parsing by passing in parameters to the `BeautifulSoup` parser via `bs_kwargs` (see @@ -247,9 +247,9 @@ In `DocumentLoader`: Object that loads data from a source as list of `Documents`. -- [Docs](../../../docs/modules/data_connection/document_loaders/): +- [Docs](/docs/modules/data_connection/document_loaders/): Detailed documentation on how to use `DocumentLoaders`. -- [Integrations](../../../docs/integrations/document_loaders/): 160+ +- [Integrations](/docs/integrations/document_loaders/): 160+ integrations to choose from. - [Interface](https://api.python.langchain.com/en/latest/document_loaders/langchain_core.document_loaders.base.BaseLoader.html): API reference  for the base interface. @@ -269,7 +269,7 @@ In this case we’ll split our documents into chunks of 1000 characters with 200 characters of overlap between chunks. The overlap helps mitigate the possibility of separating a statement from important context related to it. We use the -[RecursiveCharacterTextSplitter](../../../docs/modules/data_connection/document_transformers/recursive_text_splitter), +[RecursiveCharacterTextSplitter](/docs/modules/data_connection/document_transformers/recursive_text_splitter), which will recursively split the document using common separators like new lines until each chunk is the appropriate size. This is the recommended text splitter for generic text use cases. @@ -319,16 +319,16 @@ chunks. Subclass of `DocumentTransformer`s. - Explore `Context-aware splitters`, which keep the location (“context”) of each split in the original `Document`: - [Markdown - files](../../../docs/modules/data_connection/document_transformers/markdown_header_metadata) -- [Code (py or js)](../../../docs/integrations/document_loaders/source_code) -- [Scientific papers](../../../docs/integrations/document_loaders/grobid) + files](/docs/modules/data_connection/document_transformers/markdown_header_metadata) +- [Code (py or js)](/docs/integrations/document_loaders/source_code) +- [Scientific papers](/docs/integrations/document_loaders/grobid) - [Interface](https://api.python.langchain.com/en/latest/base/langchain_text_splitters.base.TextSplitter.html): API reference for the base interface. `DocumentTransformer`: Object that performs a transformation on a list of `Document`s. -- [Docs](../../../docs/modules/data_connection/document_transformers/): Detailed documentation on how to use `DocumentTransformers` -- [Integrations](../../../docs/integrations/document_transformers/) +- [Docs](/docs/modules/data_connection/document_transformers/): Detailed documentation on how to use `DocumentTransformers` +- [Integrations](/docs/integrations/document_transformers/) - [Interface](https://api.python.langchain.com/en/latest/documents/langchain_core.documents.transformers.BaseDocumentTransformer.html): API reference for the base interface. ## 3. Indexing: Store {#indexing-store} @@ -344,9 +344,9 @@ similarity — we measure the cosine of the angle between each pair of embeddings (which are high dimensional vectors). We can embed and store all of our document splits in a single command -using the [Chroma](../../../docs/integrations/vectorstores/chroma) +using the [Chroma](/docs/integrations/vectorstores/chroma) vector store and -[OpenAIEmbeddings](../../../docs/integrations/text_embedding/openai) +[OpenAIEmbeddings](/docs/integrations/text_embedding/openai) model. ```python @@ -361,15 +361,15 @@ vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbedd `Embeddings`: Wrapper around a text embedding model, used for converting text to embeddings. -- [Docs](../../../docs/modules/data_connection/text_embedding): Detailed documentation on how to use embeddings. -- [Integrations](../../../docs/integrations/text_embedding/): 30+ integrations to choose from. +- [Docs](/docs/modules/data_connection/text_embedding): Detailed documentation on how to use embeddings. +- [Integrations](/docs/integrations/text_embedding/): 30+ integrations to choose from. - [Interface](https://api.python.langchain.com/en/latest/embeddings/langchain_core.embeddings.Embeddings.html): API reference for the base interface. `VectorStore`: Wrapper around a vector database, used for storing and querying embeddings. -- [Docs](../../../docs/modules/data_connection/vectorstores/): Detailed documentation on how to use vector stores. -- [Integrations](../../../docs/integrations/vectorstores/): 40+ integrations to choose from. +- [Docs](/docs/modules/data_connection/vectorstores/): Detailed documentation on how to use vector stores. +- [Integrations](/docs/integrations/vectorstores/): 40+ integrations to choose from. - [Interface](https://api.python.langchain.com/en/latest/vectorstores/langchain_core.vectorstores.VectorStore.html): API reference for the base interface. This completes the **Indexing** portion of the pipeline. At this point @@ -386,12 +386,12 @@ a model, and returns an answer. First we need to define our logic for searching over documents. LangChain defines a -[Retriever](../../../docs/modules/data_connection/retrievers/) interface +[Retriever](/docs/modules/data_connection/retrievers/) interface which wraps an index that can return relevant `Documents` given a string query. The most common type of `Retriever` is the -[VectorStoreRetriever](../../../docs/modules/data_connection/retrievers/vectorstore), +[VectorStoreRetriever](/docs/modules/data_connection/retrievers/vectorstore), which uses the similarity search capabilities of a vector store to facilitate retrieval. Any `VectorStore` can easily be turned into a `Retriever` with `VectorStore.as_retriever()`: @@ -428,15 +428,15 @@ to do retrieval, too. `Retriever`: An object that returns `Document`s given a text query -- [Docs](../../../docs/modules/data_connection/retrievers/): Further +- [Docs](/docs/modules/data_connection/retrievers/): Further documentation on the interface and built-in retrieval techniques. Some of which include: - `MultiQueryRetriever` [generates variants of the input - question](../../../docs/modules/data_connection/retrievers/MultiQueryRetriever) + question](/docs/modules/data_connection/retrievers/MultiQueryRetriever) to improve retrieval hit rate. - `MultiVectorRetriever` (diagram below) instead generates [variants of the - embeddings](../../../docs/modules/data_connection/retrievers/multi_vector), + embeddings](/docs/modules/data_connection/retrievers/multi_vector), also in order to improve retrieval hit rate. - `Max marginal relevance` selects for [relevance and diversity](https://www.cs.cmu.edu/~jgc/publication/The_Use_MMR_Diversity_Based_LTMIR_1998.pdf) @@ -444,8 +444,8 @@ to do retrieval, too. context. - Documents can be filtered during vector store retrieval using metadata filters, such as with a [Self Query - Retriever](../../../docs/modules/data_connection/retrievers/self_query). -- [Integrations](../../../docs/integrations/retrievers/): Integrations + Retriever](/docs/modules/data_connection/retrievers/self_query). +- [Integrations](/docs/integrations/retrievers/): Integrations with retrieval services. - [Interface](https://api.python.langchain.com/en/latest/retrievers/langchain_core.retrievers.BaseRetriever.html): API reference for the base interface. @@ -498,7 +498,7 @@ Context: filler context Answer: ``` -We’ll use the [LCEL Runnable](../../../docs/expression_language/) +We’ll use the [LCEL Runnable](/docs/expression_language/) protocol to define the chain, allowing us to - pipe together components and functions in a transparent way - automatically trace our chain in LangSmith - get streaming, async, and batched calling out of the box @@ -539,18 +539,18 @@ trace](https://smith.langchain.com/public/1799e8db-8a6d-4eb2-84d5-46e8d7d5a99b/r `ChatModel`: An LLM-backed chat model. Takes in a sequence of messages and returns a message. -- [Docs](../../../docs/modules/model_io/chat/) -- [Integrations](../../../docs/integrations/chat/): 25+ integrations to choose from. +- [Docs](/docs/modules/model_io/chat/) +- [Integrations](/docs/integrations/chat/): 25+ integrations to choose from. - [Interface](https://api.python.langchain.com/en/latest/language_models/langchain_core.language_models.chat_models.BaseChatModel.html): API reference for the base interface. `LLM`: A text-in-text-out LLM. Takes in a string and returns a string. -- [Docs](../../../docs/modules/model_io/llms) -- [Integrations](../../../docs/integrations/llms): 75+ integrations to choose from. +- [Docs](/docs/modules/model_io/llms) +- [Integrations](/docs/integrations/llms): 75+ integrations to choose from. - [Interface](https://api.python.langchain.com/en/latest/language_models/langchain_core.language_models.llms.BaseLLM.html): API reference for the base interface. See a guide on RAG with locally-running models -[here](../../../docs/use_cases/question_answering/local_retrieval_qa). +[here](/docs/use_cases/question_answering/local_retrieval_qa). #### Customizing the prompt @@ -598,10 +598,10 @@ the above sections. Along from the **Go deeper** sources mentioned above, good next steps include: - [Return - sources](../../../docs/use_cases/question_answering/sources): Learn + sources](/docs/use_cases/question_answering/sources): Learn how to return source documents -- [Streaming](../../../docs/use_cases/question_answering/streaming): +- [Streaming](/docs/use_cases/question_answering/streaming): Learn how to stream outputs and intermediate steps - [Add chat - history](../../../docs/use_cases/question_answering/chat_history): + history](/docs/use_cases/question_answering/chat_history): Learn how to add chat history to your app diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 6fdc10f9497..23bb93bc949 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -19,7 +19,7 @@ const config = { // Set the // pathname under which your site is served // For GitHub pages deployment, it is often '//' baseUrl: "/", - + trailingSlash: true, onBrokenLinks: "throw", onBrokenMarkdownLinks: "throw", diff --git a/docs/vercel.json b/docs/vercel.json index b52dabe02e2..cb28a42f4fd 100644 --- a/docs/vercel.json +++ b/docs/vercel.json @@ -1,4 +1,5 @@ { + "trailingSlash": true, "redirects": [ { "source": "/docs/integrations/providers/optimum_intel",