From 83986ea98a9a43b4d0508e3beeda9c3b5b77b832 Mon Sep 17 00:00:00 2001 From: Predrag Gruevski <2348618+obi1kenobi@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:59:22 -0400 Subject: [PATCH] Cache poetry install + unify Python/Poetry setup for lint and test jobs. (#9625) With this PR: - All lint and test jobs use the exact same Python + Poetry installation approach, instead of lints doing it one way and tests doing it another way. - The Poetry installation itself is cached, which saves ~15s per run. - We no longer pass shell commands as workflow arguments to a workflow that just runs them in a shell. This makes our actions more resilient to shell code injection. If y'all like this approach, I can modify the scheduled tests workflow and the release workflow to use this too. --- .github/actions/poetry_setup/action.yml | 46 ++++++------------- .github/workflows/_lint.yml | 39 ++++++++-------- .github/workflows/_pydantic_compatibility.yml | 11 +++-- .github/workflows/_test.yml | 11 +++-- .github/workflows/langchain_ci.yml | 15 ++++-- .../workflows/langchain_experimental_ci.yml | 19 +++++--- 6 files changed, 72 insertions(+), 69 deletions(-) diff --git a/.github/actions/poetry_setup/action.yml b/.github/actions/poetry_setup/action.yml index 39c76bec4c1..851ddda11f4 100644 --- a/.github/actions/poetry_setup/action.yml +++ b/.github/actions/poetry_setup/action.yml @@ -15,19 +15,13 @@ inputs: description: Poetry version required: true - install-command: - description: Command run for installing dependencies - required: false - default: poetry install - cache-key: description: Cache key to use for manual handling of caching required: true working-directory: - description: Directory to run install-command in - required: false - default: "" + description: Directory whose poetry.lock file should be cached + required: true runs: using: composite @@ -38,47 +32,35 @@ runs: python-version: ${{ inputs.python-version }} - uses: actions/cache@v3 - id: cache-pip - name: Cache Pip ${{ inputs.python-version }} + id: cache-bin-poetry + name: Cache Poetry binary - Python ${{ inputs.python-version }} env: - SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4" + SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1" with: path: | - ~/.cache/pip - key: pip-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }} + /opt/pipx/venvs/poetry + /opt/pipx_bin/poetry + # This step caches the poetry installation, so make sure it's keyed on the poetry version as well. + key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }} - name: Install poetry + if: steps.cache-bin-poetry.outputs.cache-hit != 'true' shell: bash env: POETRY_VERSION: ${{ inputs.poetry-version }} PYTHON_VERSION: ${{ inputs.python-version }} run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose - - name: Check Poetry File - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - poetry check - - - name: Check lock file - shell: bash - working-directory: ${{ inputs.working-directory }} - run: | - poetry lock --check - - - uses: actions/cache@v3 - id: cache-poetry + - name: Restore pip and poetry cached dependencies + uses: actions/cache@v3 env: SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4" WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }} with: path: | + ~/.cache/pip ~/.cache/pypoetry/virtualenvs ~/.cache/pypoetry/cache ~/.cache/pypoetry/artifacts ${{ env.WORKDIR }}/.venv - key: poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }} - - - run: ${{ inputs.install-command }} - working-directory: ${{ inputs.working-directory }} - shell: bash + key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }} diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml index 5b1c35a9d8c..1a01b225a00 100644 --- a/.github/workflows/_lint.yml +++ b/.github/workflows/_lint.yml @@ -80,31 +80,32 @@ jobs: find "$WORKDIR" -name '*.py' -type f -not -newermt "$OLDEST_COMMIT_TIME" -exec touch -c -m -t '200001010000' '{}' '+' echo "oldest-commit=$OLDEST_COMMIT" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v3 - id: cache-pip - name: Cache langchain editable pip install - ${{ matrix.python-version }} - env: - SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4" - with: - path: | - ~/.cache/pip - key: pip-editable-langchain-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ matrix.python-version }} - - name: Install poetry - run: | - pipx install "poetry==$POETRY_VERSION" - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - env: - SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4" + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" with: python-version: ${{ matrix.python-version }} - cache: poetry - cache-dependency-path: | - ${{ env.WORKDIR }}/**/poetry.lock + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: lint + + - name: Check Poetry File + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry check + + - name: Check lock file + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry lock --check + - name: Install dependencies working-directory: ${{ inputs.working-directory }} run: | poetry install + - name: Install langchain editable working-directory: ${{ inputs.working-directory }} if: ${{ inputs.working-directory != 'libs/langchain' }} diff --git a/.github/workflows/_pydantic_compatibility.yml b/.github/workflows/_pydantic_compatibility.yml index 5a54c2f53c7..7d8fe26d92f 100644 --- a/.github/workflows/_pydantic_compatibility.yml +++ b/.github/workflows/_pydantic_compatibility.yml @@ -27,14 +27,19 @@ jobs: name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} uses: "./.github/actions/poetry_setup" with: python-version: ${{ matrix.python-version }} - working-directory: ${{ inputs.working-directory }} poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} cache-key: pydantic-cross-compat - install-command: poetry install + + - name: Install dependencies + shell: bash + run: poetry install + - name: Install the opposite major version of pydantic # If normal tests use pydantic v1, here we'll use v2, and vice versa. shell: bash diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 179b6303d76..76d86a2862e 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -27,14 +27,19 @@ jobs: name: Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} uses: "./.github/actions/poetry_setup" with: python-version: ${{ matrix.python-version }} - working-directory: ${{ inputs.working-directory }} poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} cache-key: core - install-command: poetry install + + - name: Install dependencies + shell: bash + run: poetry install + - name: Run core tests shell: bash run: make test diff --git a/.github/workflows/langchain_ci.yml b/.github/workflows/langchain_ci.yml index 932cce90a3e..8f1fc5d8744 100644 --- a/.github/workflows/langchain_ci.yml +++ b/.github/workflows/langchain_ci.yml @@ -64,15 +64,20 @@ jobs: name: Python ${{ matrix.python-version }} extended tests steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} uses: "./.github/actions/poetry_setup" with: python-version: ${{ matrix.python-version }} - working-directory: ${{ env.WORKDIR }} poetry-version: ${{ env.POETRY_VERSION }} + working-directory: libs/langchain cache-key: extended - install-command: | - echo "Running extended tests, installing dependencies with poetry..." - poetry install -E extended_testing + + - name: Install dependencies + shell: bash + run: | + echo "Running extended tests, installing dependencies with poetry..." + poetry install -E extended_testing + - name: Run extended tests run: make extended_tests diff --git a/.github/workflows/langchain_experimental_ci.yml b/.github/workflows/langchain_experimental_ci.yml index 9edaf6aca1a..c62ff18b354 100644 --- a/.github/workflows/langchain_experimental_ci.yml +++ b/.github/workflows/langchain_experimental_ci.yml @@ -61,18 +61,23 @@ jobs: name: test with unpublished langchain - Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} uses: "./.github/actions/poetry_setup" with: python-version: ${{ matrix.python-version }} - working-directory: ${{ env.WORKDIR }} poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ env.WORKDIR }} cache-key: unpublished-langchain - install-command: | - echo "Running tests with unpublished langchain, installing dependencies with poetry..." - poetry install - echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..." - poetry run pip install -e ../langchain + - name: Install dependencies + shell: bash + run: | + echo "Running tests with unpublished langchain, installing dependencies with poetry..." + poetry install + + echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..." + poetry run pip install -e ../langchain + - name: Run tests run: make test