Cache poetry install + unify Python/Poetry setup for lint and test jobs. (#9625)

With this PR:
- All lint and test jobs use the exact same Python + Poetry installation
approach, instead of lints doing it one way and tests doing it another
way.
- The Poetry installation itself is cached, which saves ~15s per run.
- We no longer pass shell commands as workflow arguments to a workflow
that just runs them in a shell. This makes our actions more resilient to
shell code injection.

If y'all like this approach, I can modify the scheduled tests workflow
and the release workflow to use this too.

.github/workflows/_lint.yml

@@ -80,31 +80,32 @@ jobs:
           find "$WORKDIR" -name '*.py' -type f -not -newermt "$OLDEST_COMMIT_TIME" -exec touch -c -m -t '200001010000' '{}' '+'
 
           echo "oldest-commit=$OLDEST_COMMIT" >> "$GITHUB_OUTPUT"
-      - uses: actions/cache@v3
-        id: cache-pip
-        name: Cache langchain editable pip install - ${{ matrix.python-version }}
-        env:
-          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
-        with:
-          path: |
-            ~/.cache/pip
-          key: pip-editable-langchain-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ matrix.python-version }}
-      - name: Install poetry
-        run: |
-          pipx install "poetry==$POETRY_VERSION"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        env:
-          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          cache: poetry
-          cache-dependency-path: |
-            ${{ env.WORKDIR }}/**/poetry.lock
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
       - name: Install dependencies
         working-directory: ${{ inputs.working-directory }}
         run: |
           poetry install
+
       - name: Install langchain editable
         working-directory: ${{ inputs.working-directory }}
         if: ${{ inputs.working-directory != 'libs/langchain' }}

.github/workflows/_pydantic_compatibility.yml

@@ -27,14 +27,19 @@ jobs:
     name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ inputs.working-directory }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
           cache-key: pydantic-cross-compat
-          install-command: poetry install
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
       - name: Install the opposite major version of pydantic
         # If normal tests use pydantic v1, here we'll use v2, and vice versa.
         shell: bash

.github/workflows/_test.yml

@@ -27,14 +27,19 @@ jobs:
     name: Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ inputs.working-directory }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
           cache-key: core
-          install-command: poetry install
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
       - name: Run core tests
         shell: bash
         run: make test

.github/workflows/langchain_ci.yml

@@ -64,15 +64,20 @@ jobs:
     name: Python ${{ matrix.python-version }} extended tests
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ env.WORKDIR }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: libs/langchain
           cache-key: extended
-          install-command: |
-            echo "Running extended tests, installing dependencies with poetry..."
-            poetry install -E extended_testing
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running extended tests, installing dependencies with poetry..."
+          poetry install -E extended_testing
+
       - name: Run extended tests
         run: make extended_tests

.github/workflows/langchain_experimental_ci.yml

@@ -61,18 +61,23 @@ jobs:
     name: test with unpublished langchain - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ env.WORKDIR }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ env.WORKDIR }}
           cache-key: unpublished-langchain
-          install-command: |
-            echo "Running tests with unpublished langchain, installing dependencies with poetry..."
-            poetry install
 
-            echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..."
-            poetry run pip install -e ../langchain
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests with unpublished langchain, installing dependencies with poetry..."
+          poetry install
+
+          echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..."
+          poetry run pip install -e ../langchain
+
       - name: Run tests
         run: make test