mirror of
https://github.com/hwchase17/langchain.git
synced 2025-06-23 15:19:33 +00:00
community[minor]: add mintbase loader to langchain (#20089)
- [x] **Add Near NFT loader**: "community: Load NFT near block chain
using mintbase graph API"
- [x] **PR message**:
- **Description:** a description of the change
- **Twitter handle:**Kurodenjiro
---------
Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com>
Co-authored-by: Bagatur <baskaryan@gmail.com>
This commit is contained in:
Kuro Denjiro
GitHub
parent
d7e12750df
commit
fa4124b821
130
docs/docs/integrations/document_loaders/mintbase.ipynb
Normal file
130
docs/docs/integrations/document_loaders/mintbase.ipynb
Normal file
@ -0,0 +1,130 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "vm8vn9t8DvC_"
|
||||
},
|
||||
"source": [
|
||||
"# Near Blockchain"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5WjXERXzFEhg"
|
||||
},
|
||||
"source": [
|
||||
"## Overview"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "juAmbgoWD17u"
|
||||
},
|
||||
"source": [
|
||||
"The intention of this notebook is to provide a means of testing functionality in the Langchain Document Loader for Near Blockchain.\n",
|
||||
"\n",
|
||||
"Initially this Loader supports:\n",
|
||||
"\n",
|
||||
"* Loading NFTs as Documents from NFT Smart Contracts (NEP-171 and NEP-177)\n",
|
||||
"* Near Mainnnet, Near Testnet (default is mainnet)\n",
|
||||
"* Mintbase's Graph API\n",
|
||||
"\n",
|
||||
"It can be extended if the community finds value in this loader. Specifically:\n",
|
||||
"\n",
|
||||
"* Additional APIs can be added (e.g. Tranction-related APIs)\n",
|
||||
"\n",
|
||||
"This Document Loader Requires:\n",
|
||||
"\n",
|
||||
"* A free [Mintbase API Key](https://docs.mintbase.xyz/dev/mintbase-graph/)\n",
|
||||
"\n",
|
||||
"The output takes the following format:\n",
|
||||
"\n",
|
||||
"- pageContent= Individual NFT\n",
|
||||
"- metadata={'source': 'nft.yearofchef.near', 'blockchain': 'mainnet', 'tokenId': '1846'}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load NFTs into Document Loader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get MINTBASE_API_KEY from https://docs.mintbase.xyz/dev/mintbase-graph/\n",
|
||||
"\n",
|
||||
"mintbaseApiKey = \"...\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Option 1: Ethereum Mainnet (default BlockchainType)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "J3LWHARC-Kn0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from MintbaseLoader import MintbaseDocumentLoader\n",
|
||||
"\n",
|
||||
"contractAddress = \"nft.yearofchef.near\" # Year of chef contract address\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"blockchainLoader = MintbaseDocumentLoader(\n",
|
||||
" contract_address=contractAddress, blockchain_type=\"mainnet\", api_key=\"omni-site\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"nfts = blockchainLoader.load()\n",
|
||||
"\n",
|
||||
"print(nfts[:1])\n",
|
||||
"\n",
|
||||
"for doc in blockchainLoader.lazy_load():\n",
|
||||
" print()\n",
|
||||
" print(type(doc))\n",
|
||||
" print(doc)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"collapsed_sections": [
|
||||
"5WjXERXzFEhg"
|
||||
],
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
264
libs/community/langchain_community/document_loaders/mintbase.py
Normal file
264
libs/community/langchain_community/document_loaders/mintbase.py
Normal file
@ -0,0 +1,264 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Iterator, List, Literal, Optional
|
||||
|
||||
import requests
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from langchain_community.document_loaders.base import BaseLoader
|
||||
|
||||
|
||||
class MintbaseDocumentLoader(BaseLoader):
|
||||
"""Load elements from a blockchain smart contract.
|
||||
|
||||
The supported blockchains are: Near mainnet, Near testnet.
|
||||
|
||||
If no BlockchainType is specified, the default is Near mainnet.
|
||||
|
||||
The Loader uses the Mintbase API to interact with the blockchain.
|
||||
MB_API_KEY environment variable must be set to use this loader.
|
||||
|
||||
The API returns 100 NFTs per request and can be paginated using the
|
||||
startToken parameter.
|
||||
|
||||
If get_all_tokens is set to True, the loader will get all tokens
|
||||
on the contract. Note that for contracts with a large number of tokens,
|
||||
this may take a long time (e.g. 10k tokens is 100 requests).
|
||||
Default value is false for this reason.
|
||||
|
||||
The max_execution_time (sec) can be set to limit the execution time
|
||||
of the loader.
|
||||
|
||||
Future versions of this loader can:
|
||||
- Support additional Mintbase APIs (e.g. getTokens, etc.)
|
||||
|
||||
Example:
|
||||
.. code-block:: python
|
||||
|
||||
contractAddress = "nft.yearofchef.near" # Year of chef contract address
|
||||
blockchainLoader = MintbaseDocumentLoader(
|
||||
contract_address=contractAddress, blockchain_type="mainnet",api_key="omni-site"
|
||||
)
|
||||
""" # noqa: E501
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
contract_address: str,
|
||||
*,
|
||||
blockchain_type: Literal["mainnet", "testnet"],
|
||||
api_key: str = "",
|
||||
table: str = "",
|
||||
select: str = "",
|
||||
fields: Optional[List[str]] = None,
|
||||
get_all_tokens: bool = False,
|
||||
max_execution_time: Optional[int] = None,
|
||||
):
|
||||
"""
|
||||
|
||||
Args:
|
||||
contract_address: The address of the smart contract.
|
||||
blockchainType: The blockchain type.
|
||||
api_key: The Mintbase API key.
|
||||
table: name of the table to query
|
||||
select: Conditions for querying
|
||||
fields: Information to display after query
|
||||
get_all_tokens: Whether to get all tokens on the contract.
|
||||
max_execution_time: The maximum execution time (sec).
|
||||
"""
|
||||
self.contract_address = contract_address
|
||||
self.blockchainType = blockchain_type
|
||||
self.api_key = os.environ.get("MB_API_KEY") or api_key
|
||||
self.table = "mb_views_nft_tokens" or table
|
||||
self.select = 'where: {nft_contract_id: {_eq: "contract_address"}}' or select
|
||||
self.fields = fields or [
|
||||
"base_uri",
|
||||
"burned_receipt_id",
|
||||
"burned_timestamp",
|
||||
"copies",
|
||||
"description",
|
||||
"expires_at",
|
||||
"extra",
|
||||
"issued_at",
|
||||
"last_transfer_receipt_id",
|
||||
"last_transfer_timestamp",
|
||||
"media",
|
||||
"media_hash",
|
||||
"metadata_content_flag",
|
||||
"metadata_id",
|
||||
"mint_memo",
|
||||
"minted_receipt_id",
|
||||
"minted_timestamp",
|
||||
"minter",
|
||||
"nft_contract_content_flag",
|
||||
"nft_contract_created_at",
|
||||
"nft_contract_icon",
|
||||
"nft_contract_id",
|
||||
"nft_contract_is_mintbase",
|
||||
"nft_contract_name",
|
||||
"nft_contract_owner_id",
|
||||
"nft_contract_reference",
|
||||
"nft_contract_spec",
|
||||
"nft_contract_symbol",
|
||||
"owner",
|
||||
"reference",
|
||||
"reference_blob",
|
||||
"reference_hash",
|
||||
"royalties",
|
||||
"royalties_percent",
|
||||
"splits",
|
||||
"starts_at",
|
||||
"title",
|
||||
"token_id",
|
||||
"updated_at",
|
||||
]
|
||||
|
||||
self.get_all_tokens = get_all_tokens
|
||||
self.max_execution_time = max_execution_time
|
||||
|
||||
if not self.api_key:
|
||||
raise ValueError("Mintbase API key not provided.")
|
||||
|
||||
if not re.match(
|
||||
r"^(([a-z\d]+[\-_])*[a-z\d]+\.)*([a-z\d]+[\-_])*[a-z\d]+$",
|
||||
self.contract_address,
|
||||
):
|
||||
raise ValueError(f"Invalid contract address {self.contract_address}")
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
result = []
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
# Define the GraphQL query as a multi-line string
|
||||
operations_doc = """
|
||||
query MyQuery {
|
||||
table(select) {
|
||||
fields
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
# Replace the placeholder with the actual contract address
|
||||
operations_doc = operations_doc.replace("select", self.select)
|
||||
operations_doc = operations_doc.replace(
|
||||
"contract_address", self.contract_address
|
||||
)
|
||||
operations_doc = operations_doc.replace("table", self.table)
|
||||
operations_doc = operations_doc.replace("fields", "\n".join(self.fields))
|
||||
|
||||
# Define the headers
|
||||
headers = {"mb-api-key": self.api_key, "Content-Type": "application/json"}
|
||||
# Define the POST data
|
||||
data = {
|
||||
"query": operations_doc,
|
||||
"variables": {},
|
||||
"operationName": "MyQuery",
|
||||
}
|
||||
|
||||
url = f"https://graph.mintbase.xyz/{self.blockchainType}"
|
||||
|
||||
response = requests.post(url, headers=headers, data=json.dumps(data))
|
||||
|
||||
if response.status_code != 200:
|
||||
raise ValueError(
|
||||
f"Request failed with status code {response.status_code}"
|
||||
)
|
||||
|
||||
items = response.json()["data"]["mb_views_nft_tokens"]
|
||||
|
||||
if not items:
|
||||
break
|
||||
|
||||
for item in items:
|
||||
content = str(item)
|
||||
token_id = item["token_id"]
|
||||
metadata = {
|
||||
"source": self.contract_address,
|
||||
"blockchain": self.blockchainType,
|
||||
"tokenId": token_id,
|
||||
}
|
||||
result.append(Document(page_content=content, metadata=metadata))
|
||||
|
||||
# exit after the first API call if get_all_tokens is False
|
||||
if not self.get_all_tokens:
|
||||
break
|
||||
|
||||
if (
|
||||
self.max_execution_time is not None
|
||||
and (time.time() - start_time) > self.max_execution_time
|
||||
):
|
||||
raise RuntimeError("Execution time exceeded the allowed time limit.")
|
||||
|
||||
if not result:
|
||||
raise ValueError(
|
||||
f"No NFTs found for contract address {self.contract_address}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def lazy_load(self) -> Iterator[Document]:
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
# Define the GraphQL query as a multi-line string
|
||||
operations_doc = """
|
||||
query MyQuery {
|
||||
table(select) {
|
||||
fields
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
# Replace the placeholder with the actual contract address
|
||||
operations_doc = operations_doc.replace("select", self.select)
|
||||
operations_doc = operations_doc.replace(
|
||||
"contract_address", self.contract_address
|
||||
)
|
||||
operations_doc = operations_doc.replace("table", self.table)
|
||||
operations_doc = operations_doc.replace("fields", "\n".join(self.fields))
|
||||
|
||||
# Define the headers
|
||||
headers = {"mb-api-key": self.api_key, "Content-Type": "application/json"}
|
||||
# Define the POST data
|
||||
data = {
|
||||
"query": operations_doc,
|
||||
"variables": {},
|
||||
"operationName": "MyQuery",
|
||||
}
|
||||
|
||||
url = f"https://graph.mintbase.xyz/{self.blockchainType}"
|
||||
|
||||
response = requests.post(url, headers=headers, data=json.dumps(data))
|
||||
|
||||
if response.status_code != 200:
|
||||
raise ValueError(
|
||||
f"Request failed with status code {response.status_code}"
|
||||
)
|
||||
|
||||
items = response.json()["data"]["mb_views_nft_tokens"]
|
||||
|
||||
if not items:
|
||||
break
|
||||
|
||||
for item in items:
|
||||
content = str(item)
|
||||
tokenId = item["token_id"]
|
||||
metadata = {
|
||||
"source": self.contract_address,
|
||||
"blockchain": self.blockchainType,
|
||||
"tokenId": tokenId,
|
||||
}
|
||||
yield Document(page_content=content, metadata=metadata)
|
||||
|
||||
# exit after the first API call if get_all_tokens is False
|
||||
if not self.get_all_tokens:
|
||||
break
|
||||
|
||||
if (
|
||||
self.max_execution_time is not None
|
||||
and (time.time() - start_time) > self.max_execution_time
|
||||
):
|
||||
raise RuntimeError("Execution time exceeded the allowed time limit.")
|
||||
Loading…
Reference in New Issue
Block a user