diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index d296bb915f9..a122a40058f 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -44,14 +44,6 @@ jobs: shell: bash run: make test - - name: Install integration dependencies - shell: bash - run: poetry install --with=test_integration - - - name: Check integration tests compile - shell: bash - run: poetry run pytest -m compile tests/integration_tests - - name: Ensure the tests did not create any additional files shell: bash run: | diff --git a/.github/workflows/langchain_cli_ci.yml b/.github/workflows/langchain_cli_ci.yml new file mode 100644 index 00000000000..400a0f285df --- /dev/null +++ b/.github/workflows/langchain_cli_ci.yml @@ -0,0 +1,53 @@ +--- +name: libs/cli CI + +on: + push: + branches: [ master ] + pull_request: + paths: + - '.github/actions/poetry_setup/action.yml' + - '.github/tools/**' + - '.github/workflows/_lint.yml' + - '.github/workflows/_test.yml' + - '.github/workflows/_pydantic_compatibility.yml' + - '.github/workflows/langchain_cli_ci.yml' + - 'libs/cli/**' + - 'libs/*' + workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI + +# If another push to the same PR or branch happens while this workflow is still running, +# cancel the earlier run in favor of the next run. +# +# There's no point in testing an outdated version of the code. GitHub only allows +# a limited number of job runners to be active at the same time, so it's better to cancel +# pointless jobs early so that more useful jobs can run sooner. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + POETRY_VERSION: "1.6.1" + WORKDIR: "libs/cli" + +jobs: + lint: + uses: + ./.github/workflows/_lint.yml + with: + working-directory: libs/cli + secrets: inherit + + test: + uses: + ./.github/workflows/_test.yml + with: + working-directory: libs/cli + secrets: inherit + + pydantic-compatibility: + uses: + ./.github/workflows/_pydantic_compatibility.yml + with: + working-directory: libs/cli + secrets: inherit diff --git a/libs/cli/Makefile b/libs/cli/Makefile new file mode 100644 index 00000000000..445235c22c8 --- /dev/null +++ b/libs/cli/Makefile @@ -0,0 +1,8 @@ +lint lint_diff: + poetry run poe lint + +test: + poetry run poe test + +format: + poetry run poe format diff --git a/libs/cli/langchain_cli/cli.py b/libs/cli/langchain_cli/cli.py index 7a801259658..2be2d9ea9da 100644 --- a/libs/cli/langchain_cli/cli.py +++ b/libs/cli/langchain_cli/cli.py @@ -1,10 +1,10 @@ -import typer import subprocess from typing import Optional + +import typer from typing_extensions import Annotated -from langchain_cli.namespaces import hub -from langchain_cli.namespaces import serve +from langchain_cli.namespaces import hub, serve app = typer.Typer(no_args_is_help=True, add_completion=False) app.add_typer(hub.hub, name="hub", help=hub.__doc__) diff --git a/libs/cli/langchain_cli/dev_scripts.py b/libs/cli/langchain_cli/dev_scripts.py index bd85813e1bb..7eee1e27351 100644 --- a/libs/cli/langchain_cli/dev_scripts.py +++ b/libs/cli/langchain_cli/dev_scripts.py @@ -4,6 +4,7 @@ Development Scripts for Hub Packages from fastapi import FastAPI from langserve.packages import add_package_route + from langchain_cli.utils.packages import get_package_root diff --git a/libs/cli/langchain_cli/namespaces/hub.py b/libs/cli/langchain_cli/namespaces/hub.py index 00cc3589a4e..cc4b11b77b8 100644 --- a/libs/cli/langchain_cli/namespaces/hub.py +++ b/libs/cli/langchain_cli/namespaces/hub.py @@ -2,13 +2,14 @@ Manage installable hub packages. """ -import typer -from typing import Optional -from typing_extensions import Annotated -from pathlib import Path +import re import shutil import subprocess -import re +from pathlib import Path +from typing import Optional + +import typer +from typing_extensions import Annotated hub = typer.Typer(no_args_is_help=True, add_completion=False) diff --git a/libs/cli/langchain_cli/namespaces/serve.py b/libs/cli/langchain_cli/namespaces/serve.py index abbc965c9f5..7ea443a2d14 100644 --- a/libs/cli/langchain_cli/namespaces/serve.py +++ b/libs/cli/langchain_cli/namespaces/serve.py @@ -2,21 +2,23 @@ Manage LangServe application projects. """ -import typer -from typing import Optional, List -from typing_extensions import Annotated -from pathlib import Path import shutil import subprocess +from pathlib import Path +from typing import List, Optional + +import tomli +import typer +from langserve.packages import get_langserve_export, list_packages +from typing_extensions import Annotated + +from langchain_cli.utils.events import create_events from langchain_cli.utils.git import ( copy_repo, - update_repo, parse_dependency_string, + update_repo, ) from langchain_cli.utils.packages import get_package_root -from langchain_cli.utils.events import create_events -from langserve.packages import list_packages, get_langserve_export -import tomli REPO_DIR = Path(typer.get_app_dir("langchain")) / "git_repos" @@ -101,7 +103,8 @@ def add( if len(repo) != 0: if len(dependencies) != 0: raise typer.BadParameter( - "Cannot specify both repo and dependencies. Please specify one or the other." + "Cannot specify both repo and dependencies. " + "Please specify one or the other." ) dependencies = [f"git+https://github.com/{r}" for r in repo] @@ -144,7 +147,8 @@ def add( # detect name conflict if langserve_export["package_name"] in installed_names: typer.echo( - f"Package with name {langserve_export['package_name']} already installed. Skipping...", + f"Package with name {langserve_export['package_name']} already " + "installed. Skipping...", ) continue @@ -154,7 +158,8 @@ def add( destination_path = package_dir / inner_api_path if destination_path.exists(): typer.echo( - f"Endpoint {langserve_export['package_name']} already exists. Skipping...", + f"Endpoint {langserve_export['package_name']} already exists. " + "Skipping...", ) continue copy_repo(source_path, destination_path) diff --git a/libs/cli/langchain_cli/package_template/package_template/chain.py b/libs/cli/langchain_cli/package_template/package_template/chain.py index ef4f9582a88..ef0a3aecf3c 100644 --- a/libs/cli/langchain_cli/package_template/package_template/chain.py +++ b/libs/cli/langchain_cli/package_template/package_template/chain.py @@ -1,6 +1,5 @@ -from langchain.prompts import ChatPromptTemplate from langchain.chat_models import ChatOpenAI - +from langchain.prompts import ChatPromptTemplate _prompt = ChatPromptTemplate.from_messages( [ diff --git a/libs/cli/langchain_cli/utils/events.py b/libs/cli/langchain_cli/utils/events.py index 44525b344e0..61b258d0168 100644 --- a/libs/cli/langchain_cli/utils/events.py +++ b/libs/cli/langchain_cli/utils/events.py @@ -1,6 +1,7 @@ -import urllib3 import json -from typing import List, Dict, Any, Optional, TypedDict +from typing import Any, Dict, List, Optional, TypedDict + +import urllib3 WRITE_KEY = "310apTK0HUFl4AOv" diff --git a/libs/cli/langchain_cli/utils/git.py b/libs/cli/langchain_cli/utils/git.py index ba16b94d3a7..1050d96720c 100644 --- a/libs/cli/langchain_cli/utils/git.py +++ b/libs/cli/langchain_cli/utils/git.py @@ -1,15 +1,16 @@ -from typing import Optional, TypedDict -from pathlib import Path - -import shutil +import hashlib import re +import shutil +from pathlib import Path +from typing import Optional, TypedDict + +from git import Repo + from langchain_cli.constants import ( + DEFAULT_GIT_REF, DEFAULT_GIT_REPO, DEFAULT_GIT_SUBDIRECTORY, - DEFAULT_GIT_REF, ) -import hashlib -from git import Repo class DependencySource(TypedDict): diff --git a/libs/cli/langchain_cli/utils/packages.py b/libs/cli/langchain_cli/utils/packages.py index e13cb91b9ce..97e14b6c7b6 100644 --- a/libs/cli/langchain_cli/utils/packages.py +++ b/libs/cli/langchain_cli/utils/packages.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Set, Optional +from typing import Optional, Set def get_package_root(cwd: Optional[Path] = None) -> Path: diff --git a/libs/cli/poetry.lock b/libs/cli/poetry.lock index dded71fa810..d2c9a8b6eef 100644 --- a/libs/cli/poetry.lock +++ b/libs/cli/poetry.lock @@ -641,13 +641,13 @@ files = [ [[package]] name = "langchain" -version = "0.0.323" +version = "0.0.324" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain-0.0.323-py3-none-any.whl", hash = "sha256:8c305c8d162262439b0cb73a6621c6ae8b3abde56d45c561a6b88709567cc765"}, - {file = "langchain-0.0.323.tar.gz", hash = "sha256:320116337933fdda48911e84f46c2d71e74eba647c05922a117c71669ccee9e2"}, + {file = "langchain-0.0.324-py3-none-any.whl", hash = "sha256:9be84d14e264567d52b93d0d2ba1e8cbf38c6e50a3914be02dbd9ea0fabaafd9"}, + {file = "langchain-0.0.324.tar.gz", hash = "sha256:d8dc589aa57699d51eeef8ce0507cd3faac4465ad0ff08dfb0a19e5661c3af44"}, ] [package.dependencies] @@ -656,7 +656,7 @@ anyio = "<4.0" async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""} dataclasses-json = ">=0.5.7,<0.7" jsonpatch = ">=1.33,<2.0" -langsmith = ">=0.0.43,<0.1.0" +langsmith = ">=0.0.52,<0.1.0" numpy = ">=1,<2" pydantic = ">=1,<3" PyYAML = ">=5.3" @@ -672,7 +672,7 @@ cli = ["typer (>=0.9.0,<0.10.0)"] cohere = ["cohere (>=4,<5)"] docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"] embeddings = ["sentence-transformers (>=2,<3)"] -extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "amazon-textract-caller (<2)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (>=0,<1)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "amazon-textract-caller (<2)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (>=0,<1)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] javascript = ["esprima (>=4.0.1,<5.0.0)"] llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.6.0)"] @@ -1149,6 +1149,32 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "ruff" +version = "0.1.3" +description = "An extremely fast Python linter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.3-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:b46d43d51f7061652eeadb426a9e3caa1e0002470229ab2fc19de8a7b0766901"}, + {file = "ruff-0.1.3-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:b8afeb9abd26b4029c72adc9921b8363374f4e7edb78385ffaa80278313a15f9"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca3cf365bf32e9ba7e6db3f48a4d3e2c446cd19ebee04f05338bc3910114528b"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4874c165f96c14a00590dcc727a04dca0cfd110334c24b039458c06cf78a672e"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eec2dd31eed114e48ea42dbffc443e9b7221976554a504767ceaee3dd38edeb8"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dc3ec4edb3b73f21b4aa51337e16674c752f1d76a4a543af56d7d04e97769613"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e3de9ed2e39160800281848ff4670e1698037ca039bda7b9274f849258d26ce"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c595193881922cc0556a90f3af99b1c5681f0c552e7a2a189956141d8666fe8"}, + {file = "ruff-0.1.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f75e670d529aa2288cd00fc0e9b9287603d95e1536d7a7e0cafe00f75e0dd9d"}, + {file = "ruff-0.1.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76dd49f6cd945d82d9d4a9a6622c54a994689d8d7b22fa1322983389b4892e20"}, + {file = "ruff-0.1.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:918b454bc4f8874a616f0d725590277c42949431ceb303950e87fef7a7d94cb3"}, + {file = "ruff-0.1.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d8859605e729cd5e53aa38275568dbbdb4fe882d2ea2714c5453b678dca83784"}, + {file = "ruff-0.1.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0b6c55f5ef8d9dd05b230bb6ab80bc4381ecb60ae56db0330f660ea240cb0d4a"}, + {file = "ruff-0.1.3-py3-none-win32.whl", hash = "sha256:3e7afcbdcfbe3399c34e0f6370c30f6e529193c731b885316c5a09c9e4317eef"}, + {file = "ruff-0.1.3-py3-none-win_amd64.whl", hash = "sha256:7a18df6638cec4a5bd75350639b2bb2a2366e01222825562c7346674bdceb7ea"}, + {file = "ruff-0.1.3-py3-none-win_arm64.whl", hash = "sha256:12fd53696c83a194a2db7f9a46337ce06445fb9aa7d25ea6f293cf75b21aca9f"}, + {file = "ruff-0.1.3.tar.gz", hash = "sha256:3ba6145369a151401d5db79f0a47d50e470384d0d89d0d6f7fab0b589ad07c34"}, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -1478,4 +1504,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "dd2b124e3e0b96500d6096b31fc7094840524eb85daf3d984d649e96274113fc" +content-hash = "dc768239c13daf0606a7555d611dbdaf71b00e0d3659842cd1d81eab8cfeeada" diff --git a/libs/cli/pyproject.toml b/libs/cli/pyproject.toml index 8e7f64ec5d3..8e484687672 100644 --- a/libs/cli/pyproject.toml +++ b/libs/cli/pyproject.toml @@ -18,16 +18,30 @@ uvicorn = "^0.23.2" langchain = "langchain_cli.cli:app" langchain-cli = "langchain_cli.cli:app" - [tool.poetry.group.dev.dependencies] poethepoet = "^0.24.1" pytest = "^7.4.2" pytest-watch = "^4.2.0" +[tool.poetry.group.lint.dependencies] +ruff = "^0.1.3" + +[tool.poetry.group.test.dependencies] + +[tool.poetry.group.typing.dependencies] + +[tool.ruff] +select = [ + "E", # pycodestyle + "F", # pyflakes + "I", # isort +] + [tool.poe.tasks] test = "poetry run pytest" watch = "poetry run ptw" lint = "poetry run ruff ." +format = "poetry run ruff . --fix" [build-system] diff --git a/libs/cli/tests/test_utils.py b/libs/cli/tests/test_utils.py index 1c103806aca..aa636034657 100644 --- a/libs/cli/tests/test_utils.py +++ b/libs/cli/tests/test_utils.py @@ -1,10 +1,11 @@ import pytest -from langchain_cli.utils.git import parse_dependency_string, DependencySource + from langchain_cli.constants import ( + DEFAULT_GIT_REF, DEFAULT_GIT_REPO, DEFAULT_GIT_SUBDIRECTORY, - DEFAULT_GIT_REF, ) +from langchain_cli.utils.git import DependencySource, parse_dependency_string def test_dependency_string() -> None: