CLI CI 2 (#12387)

Will run all CI because of _test change, but future PRs against CLI will
only trigger the new CLI one

Has a bunch of file changes related to formatting/linting.

No mypy yet - coming soon

.github/workflows/_test.yml

@@ -44,14 +44,6 @@ jobs:
         shell: bash
         run: make test
 
-      - name: Install integration dependencies
-        shell: bash
-        run: poetry install --with=test_integration
-
-      - name: Check integration tests compile
-        shell: bash
-        run: poetry run pytest -m compile tests/integration_tests
-
       - name: Ensure the tests did not create any additional files
         shell: bash
         run: |

.github/workflows/langchain_cli_ci.yml

@@ -0,0 +1,53 @@
+---
+name: libs/cli CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/actions/poetry_setup/action.yml'
+      - '.github/tools/**'
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/_pydantic_compatibility.yml'
+      - '.github/workflows/langchain_cli_ci.yml'
+      - 'libs/cli/**'
+      - 'libs/*'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: "libs/cli"
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/cli
+    secrets: inherit
+
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/cli
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: libs/cli
+    secrets: inherit

libs/cli/Makefile

@@ -0,0 +1,8 @@
+lint lint_diff:
+	poetry run poe lint
+
+test:
+	poetry run poe test
+
+format:
+	poetry run poe format

libs/cli/langchain_cli/cli.py

@@ -1,10 +1,10 @@
-import typer
 import subprocess
 from typing import Optional
+
+import typer
 from typing_extensions import Annotated
 
-from langchain_cli.namespaces import hub
-from langchain_cli.namespaces import serve
+from langchain_cli.namespaces import hub, serve
 
 app = typer.Typer(no_args_is_help=True, add_completion=False)
 app.add_typer(hub.hub, name="hub", help=hub.__doc__)

libs/cli/langchain_cli/dev_scripts.py

@@ -4,6 +4,7 @@ Development Scripts for Hub Packages
 
 from fastapi import FastAPI
 from langserve.packages import add_package_route
+
 from langchain_cli.utils.packages import get_package_root
 
 

libs/cli/langchain_cli/namespaces/hub.py

@@ -2,13 +2,14 @@
 Manage installable hub packages.
 """
 
-import typer
-from typing import Optional
-from typing_extensions import Annotated
-from pathlib import Path
+import re
 import shutil
 import subprocess
-import re
+from pathlib import Path
+from typing import Optional
+
+import typer
+from typing_extensions import Annotated
 
 hub = typer.Typer(no_args_is_help=True, add_completion=False)
 

libs/cli/langchain_cli/namespaces/serve.py

@@ -2,21 +2,23 @@
 Manage LangServe application projects.
 """
 
-import typer
-from typing import Optional, List
-from typing_extensions import Annotated
-from pathlib import Path
 import shutil
 import subprocess
+from pathlib import Path
+from typing import List, Optional
+
+import tomli
+import typer
+from langserve.packages import get_langserve_export, list_packages
+from typing_extensions import Annotated
+
+from langchain_cli.utils.events import create_events
 from langchain_cli.utils.git import (
     copy_repo,
-    update_repo,
     parse_dependency_string,
+    update_repo,
 )
 from langchain_cli.utils.packages import get_package_root
-from langchain_cli.utils.events import create_events
-from langserve.packages import list_packages, get_langserve_export
-import tomli
 
 REPO_DIR = Path(typer.get_app_dir("langchain")) / "git_repos"
 
@@ -101,7 +103,8 @@ def add(
     if len(repo) != 0:
         if len(dependencies) != 0:
             raise typer.BadParameter(
-                "Cannot specify both repo and dependencies. Please specify one or the other."
+                "Cannot specify both repo and dependencies. "
+                "Please specify one or the other."
             )
         dependencies = [f"git+https://github.com/{r}" for r in repo]
 
@@ -144,7 +147,8 @@ def add(
         # detect name conflict
         if langserve_export["package_name"] in installed_names:
             typer.echo(
-                f"Package with name {langserve_export['package_name']} already installed. Skipping...",
+                f"Package with name {langserve_export['package_name']} already "
+                "installed. Skipping...",
             )
             continue
 
@@ -154,7 +158,8 @@ def add(
         destination_path = package_dir / inner_api_path
         if destination_path.exists():
             typer.echo(
-                f"Endpoint {langserve_export['package_name']} already exists. Skipping...",
+                f"Endpoint {langserve_export['package_name']} already exists. "
+                "Skipping...",
             )
             continue
         copy_repo(source_path, destination_path)

libs/cli/langchain_cli/package_template/package_template/chain.py

@@ -1,6 +1,5 @@
-from langchain.prompts import ChatPromptTemplate
 from langchain.chat_models import ChatOpenAI
-
+from langchain.prompts import ChatPromptTemplate
 
 _prompt = ChatPromptTemplate.from_messages(
     [

libs/cli/langchain_cli/utils/events.py

@@ -1,6 +1,7 @@
-import urllib3
 import json
-from typing import List, Dict, Any, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict
+
+import urllib3
 
 WRITE_KEY = "310apTK0HUFl4AOv"
 

libs/cli/langchain_cli/utils/git.py

@@ -1,15 +1,16 @@
-from typing import Optional, TypedDict
-from pathlib import Path
-
-import shutil
+import hashlib
 import re
+import shutil
+from pathlib import Path
+from typing import Optional, TypedDict
+
+from git import Repo
+
 from langchain_cli.constants import (
+    DEFAULT_GIT_REF,
     DEFAULT_GIT_REPO,
     DEFAULT_GIT_SUBDIRECTORY,
-    DEFAULT_GIT_REF,
 )
-import hashlib
-from git import Repo
 
 
 class DependencySource(TypedDict):

libs/cli/langchain_cli/utils/packages.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Set, Optional
+from typing import Optional, Set
 
 
 def get_package_root(cwd: Optional[Path] = None) -> Path:

libs/cli/poetry.lock

@@ -641,13 +641,13 @@ files = [
 
 [[package]]
 name = "langchain"
-version = "0.0.323"
+version = "0.0.324"
 description = "Building applications with LLMs through composability"
 optional = false
 python-versions = ">=3.8.1,<4.0"
 files = [
-    {file = "langchain-0.0.323-py3-none-any.whl", hash = "sha256:8c305c8d162262439b0cb73a6621c6ae8b3abde56d45c561a6b88709567cc765"},
-    {file = "langchain-0.0.323.tar.gz", hash = "sha256:320116337933fdda48911e84f46c2d71e74eba647c05922a117c71669ccee9e2"},
+    {file = "langchain-0.0.324-py3-none-any.whl", hash = "sha256:9be84d14e264567d52b93d0d2ba1e8cbf38c6e50a3914be02dbd9ea0fabaafd9"},
+    {file = "langchain-0.0.324.tar.gz", hash = "sha256:d8dc589aa57699d51eeef8ce0507cd3faac4465ad0ff08dfb0a19e5661c3af44"},
 ]
 
 [package.dependencies]
@@ -656,7 +656,7 @@ anyio = "<4.0"
 async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""}
 dataclasses-json = ">=0.5.7,<0.7"
 jsonpatch = ">=1.33,<2.0"
-langsmith = ">=0.0.43,<0.1.0"
+langsmith = ">=0.0.52,<0.1.0"
 numpy = ">=1,<2"
 pydantic = ">=1,<3"
 PyYAML = ">=5.3"
@@ -672,7 +672,7 @@ cli = ["typer (>=0.9.0,<0.10.0)"]
 cohere = ["cohere (>=4,<5)"]
 docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"]
 embeddings = ["sentence-transformers (>=2,<3)"]
-extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "amazon-textract-caller (<2)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (>=0,<1)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
+extended-testing = ["aiosqlite (>=0.19.0,<0.20.0)", "amazon-textract-caller (<2)", "anthropic (>=0.3.11,<0.4.0)", "arxiv (>=1.4,<2.0)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "google-cloud-documentai (>=2.20.1,<3.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "motor (>=3.3.1,<4.0.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "numexpr (>=2.8.6,<3.0.0)", "openai (>=0,<1)", "openapi-pydantic (>=0.3.2,<0.4.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "rapidocr-onnxruntime (>=1.3.2,<2.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "rspace_client (>=2.5.0,<3.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "upstash-redis (>=0.15.0,<0.16.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
 javascript = ["esprima (>=4.0.1,<5.0.0)"]
 llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"]
 openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.6.0)"]
@@ -1149,6 +1149,32 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9
 [package.extras]
 jupyter = ["ipywidgets (>=7.5.1,<9)"]
 
+[[package]]
+name = "ruff"
+version = "0.1.3"
+description = "An extremely fast Python linter, written in Rust."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "ruff-0.1.3-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:b46d43d51f7061652eeadb426a9e3caa1e0002470229ab2fc19de8a7b0766901"},
+    {file = "ruff-0.1.3-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:b8afeb9abd26b4029c72adc9921b8363374f4e7edb78385ffaa80278313a15f9"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca3cf365bf32e9ba7e6db3f48a4d3e2c446cd19ebee04f05338bc3910114528b"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4874c165f96c14a00590dcc727a04dca0cfd110334c24b039458c06cf78a672e"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eec2dd31eed114e48ea42dbffc443e9b7221976554a504767ceaee3dd38edeb8"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dc3ec4edb3b73f21b4aa51337e16674c752f1d76a4a543af56d7d04e97769613"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e3de9ed2e39160800281848ff4670e1698037ca039bda7b9274f849258d26ce"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c595193881922cc0556a90f3af99b1c5681f0c552e7a2a189956141d8666fe8"},
+    {file = "ruff-0.1.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f75e670d529aa2288cd00fc0e9b9287603d95e1536d7a7e0cafe00f75e0dd9d"},
+    {file = "ruff-0.1.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76dd49f6cd945d82d9d4a9a6622c54a994689d8d7b22fa1322983389b4892e20"},
+    {file = "ruff-0.1.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:918b454bc4f8874a616f0d725590277c42949431ceb303950e87fef7a7d94cb3"},
+    {file = "ruff-0.1.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d8859605e729cd5e53aa38275568dbbdb4fe882d2ea2714c5453b678dca83784"},
+    {file = "ruff-0.1.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0b6c55f5ef8d9dd05b230bb6ab80bc4381ecb60ae56db0330f660ea240cb0d4a"},
+    {file = "ruff-0.1.3-py3-none-win32.whl", hash = "sha256:3e7afcbdcfbe3399c34e0f6370c30f6e529193c731b885316c5a09c9e4317eef"},
+    {file = "ruff-0.1.3-py3-none-win_amd64.whl", hash = "sha256:7a18df6638cec4a5bd75350639b2bb2a2366e01222825562c7346674bdceb7ea"},
+    {file = "ruff-0.1.3-py3-none-win_arm64.whl", hash = "sha256:12fd53696c83a194a2db7f9a46337ce06445fb9aa7d25ea6f293cf75b21aca9f"},
+    {file = "ruff-0.1.3.tar.gz", hash = "sha256:3ba6145369a151401d5db79f0a47d50e470384d0d89d0d6f7fab0b589ad07c34"},
+]
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@@ -1478,4 +1504,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0"
-content-hash = "dd2b124e3e0b96500d6096b31fc7094840524eb85daf3d984d649e96274113fc"
+content-hash = "dc768239c13daf0606a7555d611dbdaf71b00e0d3659842cd1d81eab8cfeeada"

libs/cli/pyproject.toml

@@ -18,16 +18,30 @@ uvicorn = "^0.23.2"
 langchain = "langchain_cli.cli:app"
 langchain-cli = "langchain_cli.cli:app"
 
-
 [tool.poetry.group.dev.dependencies]
 poethepoet = "^0.24.1"
 pytest = "^7.4.2"
 pytest-watch = "^4.2.0"
 
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.1.3"
+
+[tool.poetry.group.test.dependencies]
+
+[tool.poetry.group.typing.dependencies]
+
+[tool.ruff]
+select = [
+  "E",  # pycodestyle
+  "F",  # pyflakes
+  "I",  # isort
+]
+
 [tool.poe.tasks]
 test = "poetry run pytest"
 watch = "poetry run ptw"
 lint = "poetry run ruff ."
+format = "poetry run ruff . --fix"
 
 
 [build-system]

libs/cli/tests/test_utils.py

@@ -1,10 +1,11 @@
 import pytest
-from langchain_cli.utils.git import parse_dependency_string, DependencySource
+
 from langchain_cli.constants import (
+    DEFAULT_GIT_REF,
     DEFAULT_GIT_REPO,
     DEFAULT_GIT_SUBDIRECTORY,
-    DEFAULT_GIT_REF,
 )
+from langchain_cli.utils.git import DependencySource, parse_dependency_string
 
 
 def test_dependency_string() -> None: