mirror of
https://github.com/hwchase17/langchain.git
synced 2025-07-03 03:38:06 +00:00
Merge remote-tracking branch 'origin' into small_dep_fixes
This commit is contained in:
commit
e1791225ae
24
.github/actions/poetry_setup/action.yml
vendored
24
.github/actions/poetry_setup/action.yml
vendored
@ -31,20 +31,20 @@ runs:
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
|
||||
- uses: actions/cache@v3
|
||||
id: cache-bin-poetry
|
||||
name: Cache Poetry binary - Python ${{ inputs.python-version }}
|
||||
env:
|
||||
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
|
||||
with:
|
||||
path: |
|
||||
/opt/pipx/venvs/poetry
|
||||
/opt/pipx_bin/poetry
|
||||
# This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
|
||||
key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
|
||||
# - uses: actions/cache@v3
|
||||
# id: cache-bin-poetry
|
||||
# name: Cache Poetry binary - Python ${{ inputs.python-version }}
|
||||
# env:
|
||||
# SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
|
||||
# with:
|
||||
# path: |
|
||||
# /opt/pipx/venvs/poetry
|
||||
# /opt/pipx_bin/poetry
|
||||
# # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
|
||||
# key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
|
||||
|
||||
- name: Install poetry
|
||||
if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
|
||||
# if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
POETRY_VERSION: ${{ inputs.poetry-version }}
|
||||
|
@ -1,7 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import warnings
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union, cast
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
@ -129,9 +130,18 @@ class AsyncHtmlLoader(BaseLoader):
|
||||
def load(self) -> List[Document]:
|
||||
"""Load text from the url(s) in web_path."""
|
||||
|
||||
results = asyncio.run(self.fetch_all(self.web_paths))
|
||||
try:
|
||||
# Raises RuntimeError if there is no current event loop.
|
||||
asyncio.get_running_loop()
|
||||
# If there is a current event loop, we need to run the async code
|
||||
# in a separate loop, in a separate thread.
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(asyncio.run, self.fetch_all(self.web_paths))
|
||||
results = future.result()
|
||||
except RuntimeError:
|
||||
results = asyncio.run(self.fetch_all(self.web_paths))
|
||||
docs = []
|
||||
for i, text in enumerate(results):
|
||||
for i, text in enumerate(cast(List[str], results)):
|
||||
metadata = {"source": self.web_paths[i]}
|
||||
docs.append(Document(page_content=text, metadata=metadata))
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "langchain"
|
||||
version = "0.0.281"
|
||||
version = "0.0.282"
|
||||
description = "Building applications with LLMs through composability"
|
||||
authors = []
|
||||
license = "MIT"
|
||||
|
Loading…
Reference in New Issue
Block a user