Cache poetry install + unify Python/Poetry setup for lint and test jobs. (#9625)

With this PR:
- All lint and test jobs use the exact same Python + Poetry installation
approach, instead of lints doing it one way and tests doing it another
way.
- The Poetry installation itself is cached, which saves ~15s per run.
- We no longer pass shell commands as workflow arguments to a workflow
that just runs them in a shell. This makes our actions more resilient to
shell code injection.

If y'all like this approach, I can modify the scheduled tests workflow
and the release workflow to use this too.

.github/actions/poetry_setup/action.yml

@@ -15,19 +15,13 @@ inputs:
     description: Poetry version
     required: true
 
-  install-command:
-    description: Command run for installing dependencies
-    required: false
-    default: poetry install
-
   cache-key:
     description: Cache key to use for manual handling of caching
     required: true
 
   working-directory:
-    description: Directory to run install-command in
-    required: false
-    default: ""
+    description: Directory whose poetry.lock file should be cached
+    required: true
 
 runs:
   using: composite
@@ -38,47 +32,35 @@ runs:
         python-version: ${{ inputs.python-version }}
 
     - uses: actions/cache@v3
-      id: cache-pip
-      name: Cache Pip ${{ inputs.python-version }}
+      id: cache-bin-poetry
+      name: Cache Poetry binary - Python ${{ inputs.python-version }}
       env:
-        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
       with:
         path: |
-          ~/.cache/pip
-        key: pip-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}
+          /opt/pipx/venvs/poetry
+          /opt/pipx_bin/poetry
+        # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+        key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
 
     - name: Install poetry
+      if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
       shell: bash
       env:
         POETRY_VERSION: ${{ inputs.poetry-version }}
         PYTHON_VERSION: ${{ inputs.python-version }}
       run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
 
-    - name: Check Poetry File
-      shell: bash
-      working-directory: ${{ inputs.working-directory }}
-      run: |
-        poetry check
-
-    - name: Check lock file
-      shell: bash
-      working-directory: ${{ inputs.working-directory }}
-      run: |
-        poetry lock --check
-
-    - uses: actions/cache@v3
-      id: cache-poetry
+    - name: Restore pip and poetry cached dependencies
+      uses: actions/cache@v3
       env:
         SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
         WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
       with:
         path: |
+          ~/.cache/pip
           ~/.cache/pypoetry/virtualenvs
           ~/.cache/pypoetry/cache
           ~/.cache/pypoetry/artifacts
           ${{ env.WORKDIR }}/.venv
-        key: poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
-
-    - run: ${{ inputs.install-command }}
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
+        key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_lint.yml

@@ -80,31 +80,32 @@ jobs:
           find "$WORKDIR" -name '*.py' -type f -not -newermt "$OLDEST_COMMIT_TIME" -exec touch -c -m -t '200001010000' '{}' '+'
 
           echo "oldest-commit=$OLDEST_COMMIT" >> "$GITHUB_OUTPUT"
-      - uses: actions/cache@v3
-        id: cache-pip
-        name: Cache langchain editable pip install - ${{ matrix.python-version }}
-        env:
-          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
-        with:
-          path: |
-            ~/.cache/pip
-          key: pip-editable-langchain-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ matrix.python-version }}
-      - name: Install poetry
-        run: |
-          pipx install "poetry==$POETRY_VERSION"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        env:
-          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          cache: poetry
-          cache-dependency-path: |
-            ${{ env.WORKDIR }}/**/poetry.lock
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
       - name: Install dependencies
         working-directory: ${{ inputs.working-directory }}
         run: |
           poetry install
+
       - name: Install langchain editable
         working-directory: ${{ inputs.working-directory }}
         if: ${{ inputs.working-directory != 'libs/langchain' }}

.github/workflows/_pydantic_compatibility.yml

@@ -27,14 +27,19 @@ jobs:
     name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ inputs.working-directory }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
           cache-key: pydantic-cross-compat
-          install-command: poetry install
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
       - name: Install the opposite major version of pydantic
         # If normal tests use pydantic v1, here we'll use v2, and vice versa.
         shell: bash

.github/workflows/_test.yml

@@ -27,14 +27,19 @@ jobs:
     name: Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ inputs.working-directory }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
           cache-key: core
-          install-command: poetry install
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
       - name: Run core tests
         shell: bash
         run: make test

.github/workflows/langchain_ci.yml

@@ -64,15 +64,20 @@ jobs:
     name: Python ${{ matrix.python-version }} extended tests
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ env.WORKDIR }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: libs/langchain
           cache-key: extended
-          install-command: |
-            echo "Running extended tests, installing dependencies with poetry..."
-            poetry install -E extended_testing
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running extended tests, installing dependencies with poetry..."
+          poetry install -E extended_testing
+
       - name: Run extended tests
         run: make extended_tests

.github/workflows/langchain_experimental_ci.yml

@@ -61,18 +61,23 @@ jobs:
     name: test with unpublished langchain - Python ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_setup"
         with:
           python-version: ${{ matrix.python-version }}
-          working-directory: ${{ env.WORKDIR }}
           poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ env.WORKDIR }}
           cache-key: unpublished-langchain
-          install-command: |
-            echo "Running tests with unpublished langchain, installing dependencies with poetry..."
-            poetry install
 
-            echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..."
-            poetry run pip install -e ../langchain
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests with unpublished langchain, installing dependencies with poetry..."
+          poetry install
+
+          echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..."
+          poetry run pip install -e ../langchain
+
       - name: Run tests
         run: make test