mirror of
https://github.com/hwchase17/langchain.git
synced 2025-08-24 03:52:08 +00:00
058a64c563
Hi 👋 First off, thanks a ton for your work on this 💚 Really appreciate what you're providing here for the community. ## Description This PR adds a basic language parser for the [Elixir](https://elixir-lang.org/) programming language. The parser code is based upon the approach outlined in https://github.com/langchain-ai/langchain/pull/13318: it's using `tree-sitter` under the hood and aligns with all the other `tree-sitter` based parses added that PR. The `CHUNK_QUERY` I'm using here is probably not the most sophisticated one, but it worked for my application. It's a starting point to provide "core" parsing support for Elixir in LangChain. It enables people to use the language parser out in real world applications which may then lead to further tweaking of the queries. I consider this PR just the ground work. - **Dependencies:** requires `tree-sitter` and `tree-sitter-languages` from the extended dependencies - **Twitter handle:**`@bitcrowd` ## Checklist - [x] **PR title**: "package: description" - [x] **Add tests and docs** - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. <!-- If no one reviews your PR within a few days, please @-mention one of baskaryan, efriis, eyurtsev, ccurme, vbarda, hwchase17. -->
36 lines
1.0 KiB
Python
36 lines
1.0 KiB
Python
from typing import TYPE_CHECKING
|
|
|
|
from langchain_community.document_loaders.parsers.language.tree_sitter_segmenter import ( # noqa: E501
|
|
TreeSitterSegmenter,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from tree_sitter import Language
|
|
|
|
|
|
CHUNK_QUERY = """
|
|
[
|
|
(call target: ((identifier) @_identifier
|
|
(#any-of? @_identifier "defmodule" "defprotocol" "defimpl"))) @module
|
|
(call target: ((identifier) @_identifier
|
|
(#any-of? @_identifier "def" "defmacro" "defmacrop" "defp"))) @function
|
|
(unary_operator operator: "@" operand: (call target: ((identifier) @_identifier
|
|
(#any-of? @_identifier "moduledoc" "typedoc""doc")))) @comment
|
|
]
|
|
""".strip()
|
|
|
|
|
|
class ElixirSegmenter(TreeSitterSegmenter):
|
|
"""Code segmenter for Elixir."""
|
|
|
|
def get_language(self) -> "Language":
|
|
from tree_sitter_languages import get_language
|
|
|
|
return get_language("elixir")
|
|
|
|
def get_chunk_query(self) -> str:
|
|
return CHUNK_QUERY
|
|
|
|
def make_line_comment(self, text: str) -> str:
|
|
return f"# {text}"
|