diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py index 3aa0ceb04fb..f02d59acad9 100644 --- a/.github/scripts/check_diff.py +++ b/.github/scripts/check_diff.py @@ -1,17 +1,23 @@ import json import sys import os +from typing import Dict -LANGCHAIN_DIRS = { +LANGCHAIN_DIRS = [ "libs/core", "libs/langchain", "libs/experimental", "libs/community", -} +] if __name__ == "__main__": files = sys.argv[1:] - dirs_to_run = set() + + dirs_to_run: Dict[str, set] = { + "lint": set(), + "test": set(), + "extended-test": set(), + } if len(files) == 300: # max diff length is 300 files - there are likely files missing @@ -24,31 +30,42 @@ if __name__ == "__main__": ".github/workflows", ".github/tools", ".github/actions", - "libs/core", ".github/scripts/check_diff.py", ) ): - dirs_to_run.update(LANGCHAIN_DIRS) - elif "libs/community" in file: - dirs_to_run.update( - ("libs/community", "libs/langchain", "libs/experimental") - ) - elif "libs/partners" in file: + # add all LANGCHAIN_DIRS for infra changes + dirs_to_run["extended-test"].update(LANGCHAIN_DIRS) + dirs_to_run["lint"].add(".") + + if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS): + # add that dir and all dirs after in LANGCHAIN_DIRS + # for extended testing + found = False + for dir_ in LANGCHAIN_DIRS: + if file.startswith(dir_): + found = True + if found: + dirs_to_run["extended-test"].add(dir_) + elif file.startswith("libs/partners"): partner_dir = file.split("/")[2] if os.path.isdir(f"libs/partners/{partner_dir}"): - dirs_to_run.add(f"libs/partners/{partner_dir}") + dirs_to_run["test"].add(f"libs/partners/{partner_dir}") # Skip if the directory was deleted - elif "libs/langchain" in file: - dirs_to_run.update(("libs/langchain", "libs/experimental")) - elif "libs/experimental" in file: - dirs_to_run.add("libs/experimental") elif file.startswith("libs/"): - dirs_to_run.update(LANGCHAIN_DIRS) - else: - pass - json_output = json.dumps(list(dirs_to_run)) - print(f"dirs-to-run={json_output}") # noqa: T201 + raise ValueError( + f"Unknown lib: {file}. check_diff.py likely needs " + "an update for this new library!" + ) + elif any(file.startswith(p) for p in ["docs/", "templates/", "cookbook/"]): + dirs_to_run["lint"].add(".") - extended_test_dirs = [d for d in dirs_to_run if not d.startswith("libs/partners")] - json_output_extended = json.dumps(extended_test_dirs) - print(f"dirs-to-run-extended={json_output_extended}") # noqa: T201 + outputs = { + "dirs-to-lint": list( + dirs_to_run["lint"] | dirs_to_run["test"] | dirs_to_run["extended-test"] + ), + "dirs-to-test": list(dirs_to_run["test"] | dirs_to_run["extended-test"]), + "dirs-to-extended-test": list(dirs_to_run["extended-test"]), + } + for key, value in outputs.items(): + json_output = json.dumps(value) + print(f"{key}={json_output}") # noqa: T201 diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml index 2fdf56c1e28..c4bd8b448b8 100644 --- a/.github/workflows/check_diffs.yml +++ b/.github/workflows/check_diffs.yml @@ -33,14 +33,16 @@ jobs: run: | python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT outputs: - dirs-to-run: ${{ steps.set-matrix.outputs.dirs-to-run }} - dirs-to-run-extended: ${{ steps.set-matrix.outputs.dirs-to-run-extended }} + dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }} + dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }} + dirs-to-extended-test: ${{ steps.set-matrix.outputs.dirs-to-extended-test }} lint: name: cd ${{ matrix.working-directory }} needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-lint != '[]' }} strategy: matrix: - working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-lint) }} uses: ./.github/workflows/_lint.yml with: working-directory: ${{ matrix.working-directory }} @@ -49,9 +51,10 @@ jobs: test: name: cd ${{ matrix.working-directory }} needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-test != '[]' }} strategy: matrix: - working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }} uses: ./.github/workflows/_test.yml with: working-directory: ${{ matrix.working-directory }} @@ -60,9 +63,10 @@ jobs: compile-integration-tests: name: cd ${{ matrix.working-directory }} needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-test != '[]' }} strategy: matrix: - working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }} uses: ./.github/workflows/_compile_integration_test.yml with: working-directory: ${{ matrix.working-directory }} @@ -71,9 +75,10 @@ jobs: dependencies: name: cd ${{ matrix.working-directory }} needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-test != '[]' }} strategy: matrix: - working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run) }} + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }} uses: ./.github/workflows/_dependencies.yml with: working-directory: ${{ matrix.working-directory }} @@ -82,11 +87,11 @@ jobs: extended-tests: name: "cd ${{ matrix.working-directory }} / make extended_tests #${{ matrix.python-version }}" needs: [ build ] - if: ${{ needs.build.outputs.dirs-to-run-extended != '[]' }} + if: ${{ needs.build.outputs.dirs-to-extended-test != '[]' }} strategy: matrix: # note different variable for extended test dirs - working-directory: ${{ fromJson(needs.build.outputs.dirs-to-run-extended) }} + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-extended-test) }} python-version: - "3.8" - "3.9" diff --git a/.github/workflows/doc_lint.yml b/.github/workflows/doc_lint.yml deleted file mode 100644 index 298e1509b01..00000000000 --- a/.github/workflows/doc_lint.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: CI / cd . - -on: - push: - branches: [ master ] - pull_request: - paths: - - 'docs/**' - - 'templates/**' - - 'cookbook/**' - - '.github/workflows/_lint.yml' - - '.github/workflows/doc_lint.yml' - workflow_dispatch: - -jobs: - check: - name: Check for "from langchain import x" imports - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Run import check - run: | - # We should not encourage imports directly from main init file - # Expect for hub - git grep 'from langchain import' {docs/docs,templates,cookbook} | grep -vE 'from langchain import (hub)' && exit 1 || exit 0 - - lint: - name: "-" - uses: - ./.github/workflows/_lint.yml - with: - working-directory: "." - secrets: inherit \ No newline at end of file diff --git a/Makefile b/Makefile index db50dfeed32..5e66e6c07fb 100644 --- a/Makefile +++ b/Makefile @@ -50,11 +50,13 @@ lint lint_package lint_tests: poetry run ruff docs templates cookbook poetry run ruff format docs templates cookbook --diff poetry run ruff --select I docs templates cookbook + git grep 'from langchain import' {docs/docs,templates,cookbook} | grep -vE 'from langchain import (hub)' && exit 1 || exit 0 format format_diff: poetry run ruff format docs templates cookbook poetry run ruff --select I --fix docs templates cookbook + ###################### # HELP ######################