diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py index 4cbe1ec8b2f..3aa0ceb04fb 100644 --- a/.github/scripts/check_diff.py +++ b/.github/scripts/check_diff.py @@ -48,3 +48,7 @@ if __name__ == "__main__": pass json_output = json.dumps(list(dirs_to_run)) print(f"dirs-to-run={json_output}") # noqa: T201 + + extended_test_dirs = [d for d in dirs_to_run if not d.startswith("libs/partners")] + json_output_extended = json.dumps(extended_test_dirs) + print(f"dirs-to-run-extended={json_output_extended}") # noqa: T201 diff --git a/.github/workflows/_all_ci.yml b/.github/workflows/_all_ci.yml deleted file mode 100644 index 1531071885e..00000000000 --- a/.github/workflows/_all_ci.yml +++ /dev/null @@ -1,126 +0,0 @@ ---- -name: langchain CI - -on: - workflow_call: - inputs: - working-directory: - required: true - type: string - description: "From which folder this pipeline executes" - workflow_dispatch: - inputs: - working-directory: - required: true - type: choice - default: 'libs/langchain' - options: - - libs/langchain - - libs/core - - libs/experimental - - libs/community - - -# If another push to the same PR or branch happens while this workflow is still running, -# cancel the earlier run in favor of the next run. -# -# There's no point in testing an outdated version of the code. GitHub only allows -# a limited number of job runners to be active at the same time, so it's better to cancel -# pointless jobs early so that more useful jobs can run sooner. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.working-directory }} - cancel-in-progress: true - -env: - POETRY_VERSION: "1.7.1" - -jobs: - lint: - name: "-" - uses: ./.github/workflows/_lint.yml - with: - working-directory: ${{ inputs.working-directory }} - secrets: inherit - - test: - name: "-" - uses: ./.github/workflows/_test.yml - with: - working-directory: ${{ inputs.working-directory }} - secrets: inherit - - compile-integration-tests: - name: "-" - uses: ./.github/workflows/_compile_integration_test.yml - with: - working-directory: ${{ inputs.working-directory }} - secrets: inherit - - dependencies: - name: "-" - uses: ./.github/workflows/_dependencies.yml - with: - working-directory: ${{ inputs.working-directory }} - secrets: inherit - - extended-tests: - name: "make extended_tests #${{ matrix.python-version }}" - runs-on: ubuntu-latest - strategy: - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - defaults: - run: - working-directory: ${{ inputs.working-directory }} - if: ${{ ! startsWith(inputs.working-directory, 'libs/partners/') }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} - uses: "./.github/actions/poetry_setup" - with: - python-version: ${{ matrix.python-version }} - poetry-version: ${{ env.POETRY_VERSION }} - working-directory: ${{ inputs.working-directory }} - cache-key: extended - - - name: Install dependencies - shell: bash - run: | - echo "Running extended tests, installing dependencies with poetry..." - poetry install -E extended_testing --with test - - - name: Run extended tests - run: make extended_tests - - - name: Ensure the tests did not create any additional files - shell: bash - run: | - set -eu - - STATUS="$(git status)" - echo "$STATUS" - - # grep will exit non-zero if the target message isn't found, - # and `set -e` above will cause the step to fail. - echo "$STATUS" | grep 'nothing to commit, working tree clean' - ci_end: - name: "CI Success" - needs: [lint, test, compile-integration-tests, dependencies, extended-tests] - if: ${{ always() }} - runs-on: ubuntu-latest - steps: - - name: "CI Success" - if: ${{ !failure() }} - run: | - echo "Success" - exit 0 - - name: "CI Failure" - if: ${{ failure() }} - run: | - echo "Failure" - exit 1 diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml index 9a2e11dd3a0..f13e8648cbc 100644 --- a/.github/workflows/check_diffs.yml +++ b/.github/workflows/check_diffs.yml @@ -16,6 +16,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + POETRY_VERSION: "1.7.1" + jobs: build: runs-on: ubuntu-latest @@ -31,14 +34,114 @@ jobs: python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT outputs: dirs-to-run: ${{ steps.set-matrix.outputs.dirs-to-run }} - ci: + dirs-to-run-extended: ${{ steps.set-matrix.outputs.dirs-to-run-extended }} + lint: name: cd ${{ matrix.working-directory }} needs: [ build ] strategy: matrix: working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} - uses: ./.github/workflows/_all_ci.yml + uses: ./.github/workflows/_lint.yml with: working-directory: ${{ matrix.working-directory }} + secrets: inherit + + test: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + uses: ./.github/workflows/_test.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + compile-integration-tests: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + uses: ./.github/workflows/_compile_integration_test.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + dependencies: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + uses: ./.github/workflows/_dependencies.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + extended-tests: + name: "cd ${{ matrix.working-directory }} / make extended_tests #${{ matrix.python-version }}" + needs: [ build ] + strategy: + matrix: + # note different variable for extended test dirs + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run-extended) }} + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runs-on: ubuntu-latest + defaults: + run: + working-directory: ${{ matrix.working-directory }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ matrix.python-version }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ matrix.working-directory }} + cache-key: extended + + - name: Install dependencies + shell: bash + run: | + echo "Running extended tests, installing dependencies with poetry..." + poetry install -E extended_testing --with test + + - name: Run extended tests + run: make extended_tests + + - name: Ensure the tests did not create any additional files + shell: bash + run: | + set -eu + + STATUS="$(git status)" + echo "$STATUS" + + # grep will exit non-zero if the target message isn't found, + # and `set -e` above will cause the step to fail. + echo "$STATUS" | grep 'nothing to commit, working tree clean' + ci_end: + name: "CI Success" + needs: [build, lint, test, compile-integration-tests, dependencies, extended-tests] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: "CI Success" + if: ${{ !failure() }} + run: | + echo "Success" + exit 0 + - name: "CI Failure" + if: ${{ failure() }} + run: | + echo "Failure" + exit 1 +