Add PythonLoader which auto-detects encoding of Python files (#3311)

This PR contributes a `PythonLoader`, which inherits from
`TextLoader` but detects and sets the encoding automatically.
This commit is contained in:
Paul Garner
2023-04-21 18:47:57 +01:00
committed by GitHub
parent 1ecbeec24e
commit aa9d5707e0
7 changed files with 104 additions and 3 deletions

View File

@@ -11,7 +11,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 3,
"id": "019d8520",
"metadata": {},
"outputs": [],
@@ -128,10 +128,69 @@
"len(docs)"
]
},
{
"cell_type": "markdown",
"id": "598a2805",
"metadata": {},
"source": [
"If you need to load Python source code files, use the `PythonLoader`."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "c558bd73",
"metadata": {},
"outputs": [],
"source": [
"from langchain.document_loaders import PythonLoader"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "a3cfaba7",
"metadata": {},
"outputs": [],
"source": [
"loader = DirectoryLoader('../../../../../', glob=\"**/*.py\", loader_cls=PythonLoader)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "e2e1e26a",
"metadata": {},
"outputs": [],
"source": [
"docs = loader.load()"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "ffb8ff36",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"691"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(docs)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "984c8429",
"id": "7f6e0eae",
"metadata": {},
"outputs": [],
"source": []
@@ -153,7 +212,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
"version": "3.10.3"
}
},
"nbformat": 4,