Merge remote-tracking branch 'origin' into small_dep_fixes

This commit is contained in:
olgavrou 2023-09-04 22:49:16 -04:00
commit e1791225ae
3 changed files with 26 additions and 16 deletions

View File

@ -31,20 +31,20 @@ runs:
with:
python-version: ${{ inputs.python-version }}
- uses: actions/cache@v3
id: cache-bin-poetry
name: Cache Poetry binary - Python ${{ inputs.python-version }}
env:
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
with:
path: |
/opt/pipx/venvs/poetry
/opt/pipx_bin/poetry
# This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
# - uses: actions/cache@v3
# id: cache-bin-poetry
# name: Cache Poetry binary - Python ${{ inputs.python-version }}
# env:
# SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
# with:
# path: |
# /opt/pipx/venvs/poetry
# /opt/pipx_bin/poetry
# # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
# key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
- name: Install poetry
if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
# if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
shell: bash
env:
POETRY_VERSION: ${{ inputs.poetry-version }}

View File

@ -1,7 +1,8 @@
import asyncio
import logging
import warnings
from typing import Any, Dict, Iterator, List, Optional, Union
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, Iterator, List, Optional, Union, cast
import aiohttp
import requests
@ -129,9 +130,18 @@ class AsyncHtmlLoader(BaseLoader):
def load(self) -> List[Document]:
"""Load text from the url(s) in web_path."""
results = asyncio.run(self.fetch_all(self.web_paths))
try:
# Raises RuntimeError if there is no current event loop.
asyncio.get_running_loop()
# If there is a current event loop, we need to run the async code
# in a separate loop, in a separate thread.
with ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(asyncio.run, self.fetch_all(self.web_paths))
results = future.result()
except RuntimeError:
results = asyncio.run(self.fetch_all(self.web_paths))
docs = []
for i, text in enumerate(results):
for i, text in enumerate(cast(List[str], results)):
metadata = {"source": self.web_paths[i]}
docs.append(Document(page_content=text, metadata=metadata))

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "langchain"
version = "0.0.281"
version = "0.0.282"
description = "Building applications with LLMs through composability"
authors = []
license = "MIT"