langchain/.github/workflows/integration_tests.yml

# Routine integration tests against partner libraries with live API credentials.
#
# Uses `make integration_tests` within each library being tested.
#
# Runs daily with the option to trigger manually.

name: "⏰ Integration Tests"
run-name: "Run Integration Tests - ${{ inputs.working-directory-override || (inputs.working-directory != 'all' && inputs.working-directory) || (inputs.exclude != '' && format('exclude:{0}', inputs.exclude)) || 'all libs' }} (Python ${{ inputs.python-version-override || '3.10, 3.13' }})"

on:
  workflow_dispatch:
    inputs:
      working-directory:
        type: choice
        description: "Library to test (select from dropdown)"
        default: "all"
        # Short names only — the `compute-matrix` job re-adds the `libs/` or
        # `libs/partners/` prefix. When adding a new option, also update the
        # `case` statement in `compute-matrix` if it isn't a partner package
        # (partners are the default branch).
        options:
          - "all"
          - "core"
          - "langchain"
          - "langchain_v1"
          - "text-splitters"
          - "standard-tests"
          - "model-profiles"
          - "anthropic"
          - "aws"
          - "chroma"
          - "deepseek"
          - "exa"
          - "fireworks"
          - "google-genai"
          - "google-vertexai"
          - "groq"
          - "huggingface"
          - "mistralai"
          - "nomic"
          - "ollama"
          - "openai"
          - "openrouter"
          - "perplexity"
          - "qdrant"
          - "xai"
      working-directory-override:
        type: string
        description: "Manual override — takes precedence over dropdown (e.g. libs/partners/partner-xyz)"
      exclude:
        type: string
        description: "Comma-separated short names to drop from the 'all' run (e.g. openai,anthropic). Ignored unless dropdown is 'all' and no working-directory-override is set."
      python-version-override:
        type: string
        description: "Python version override — defaults to 3.10 and 3.13 in matrix (e.g. 3.11)"
  schedule:
    - cron: "0 13 * * *" # Runs daily at 1PM UTC (9AM EDT/6AM PDT)

permissions:
  contents: read

env:
  UV_FROZEN: "true"
  DEFAULT_LIBS: >-
    ["libs/partners/openai",
    "libs/partners/anthropic",
    "libs/partners/fireworks",
    "libs/partners/groq",
    "libs/partners/mistralai",
    "libs/partners/xai",
    "libs/partners/google-vertexai",
    "libs/partners/google-genai",
    "libs/partners/aws"]

jobs:
  # Generate dynamic test matrix based on input parameters or defaults
  # Only runs on the main repo (for scheduled runs) or when manually triggered
  compute-matrix:
    # Defend against forks running scheduled jobs, but allow manual runs from forks
    if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule'

    runs-on: ubuntu-latest
    name: "📋 Compute Test Matrix"
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      python-version-min-3-11: ${{ steps.set-matrix.outputs.python-version-min-3-11 }}
    steps:
      - name: "🔢 Generate Python & Library Matrix"
        id: set-matrix
        env:
          DEFAULT_LIBS: ${{ env.DEFAULT_LIBS }}
          WORKING_DIRECTORY_OVERRIDE: ${{ github.event.inputs.working-directory-override || '' }}
          WORKING_DIRECTORY_CHOICE: ${{ github.event.inputs.working-directory || 'all' }}
          PYTHON_VERSION_OVERRIDE: ${{ github.event.inputs.python-version-override || '' }}
          EXCLUDE: ${{ github.event.inputs.exclude || '' }}
        run: |
          # echo "matrix=..." where matrix is a json formatted str with keys python-version and working-directory
          # python-version defaults to 3.10 and 3.13, overridden to [PYTHON_VERSION_OVERRIDE] if set
          # working-directory priority: override string > dropdown choice > DEFAULT_LIBS
          python_version='["3.10", "3.13"]'
          python_version_min_3_11='["3.11", "3.13"]'
          working_directory="$DEFAULT_LIBS"
          if [ -n "$PYTHON_VERSION_OVERRIDE" ]; then
            python_version="[\"$PYTHON_VERSION_OVERRIDE\"]"
            # Bound override version to >= 3.11 for packages requiring it
            if [ "$(echo "$PYTHON_VERSION_OVERRIDE >= 3.11" | bc -l)" -eq 1 ]; then
              python_version_min_3_11="[\"$PYTHON_VERSION_OVERRIDE\"]"
            else
              python_version_min_3_11='["3.11"]'
            fi
          fi
          if [ -n "$WORKING_DIRECTORY_OVERRIDE" ]; then
            working_directory="[\"$WORKING_DIRECTORY_OVERRIDE\"]"
          elif [ "$WORKING_DIRECTORY_CHOICE" != "all" ]; then
            # Map short dropdown name back to full path
            case "$WORKING_DIRECTORY_CHOICE" in
              core|langchain|langchain_v1|text-splitters|standard-tests|model-profiles)
                working_directory="[\"libs/$WORKING_DIRECTORY_CHOICE\"]"
                ;;
              *)
                working_directory="[\"libs/partners/$WORKING_DIRECTORY_CHOICE\"]"
                ;;
            esac
          elif [ -n "$EXCLUDE" ]; then
            # Only honored on the 'all' run (no override, dropdown left at 'all').
            # Map each comma-separated short name to its full path (mirroring the
            # case statement above), then subtract from the DEFAULT_LIBS array.
            exclude_paths='[]'
            IFS=',' read -ra exclude_names <<< "$EXCLUDE"
            for name in "${exclude_names[@]}"; do
              # Trim surrounding whitespace so "openai, anthropic" works.
              name="${name#"${name%%[![:space:]]*}"}" # ltrim
              name="${name%"${name##*[![:space:]]}"}" # rtrim
              [ -z "$name" ] && continue
              case "$name" in
                core|langchain|langchain_v1|text-splitters|standard-tests|model-profiles)
                  path="libs/$name"
                  ;;
                *)
                  path="libs/partners/$name"
                  ;;
              esac
              exclude_paths="$(jq -nc --argjson acc "$exclude_paths" --arg path "$path" '$acc + [$path]')"
            done
            working_directory="$(jq -nc --argjson libs "$working_directory" --argjson excl "$exclude_paths" '$libs - $excl')"
          fi
          matrix="{\"python-version\": $python_version, \"working-directory\": $working_directory}"
          echo "$matrix"
          echo "matrix=$matrix" >> $GITHUB_OUTPUT
          echo "python-version-min-3-11=$python_version_min_3_11" >> $GITHUB_OUTPUT

  # Run integration tests against partner libraries with live API credentials
  integration-tests:
    if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule'
    name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.working-directory }}"
    runs-on: ubuntu-latest
    # Scopes LangSmith tracing credentials (and any other env-scoped secrets)
    environment: "Scheduled testing"
    needs: [compute-matrix]
    timeout-minutes: 30
    # Serialize same-package shards across workflow runs so a per-package
    # manual dispatch doesn't race the scheduled "all libs" run against the
    # same live API credentials. Keyed per (working-directory, python-version)
    # so the 3.10/3.13 matrix legs within one run still execute in parallel.
    concurrency:
      group: integration-tests-${{ matrix.working-directory }}-${{ matrix.python-version }}
      cancel-in-progress: false
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJSON(needs.compute-matrix.outputs.matrix).python-version }}
        working-directory: ${{ fromJSON(needs.compute-matrix.outputs.matrix).working-directory }}

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          path: langchain

      # These libraries exist outside of the monorepo and need to be checked out separately
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: langchain-ai/langchain-google
          path: langchain-google
      - name: "🔐 Authenticate to Google Cloud"
        id: "auth"
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3
        with:
          credentials_json: "${{ secrets.GOOGLE_CREDENTIALS }}"
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: langchain-ai/langchain-aws
          path: langchain-aws
      - name: "🔐 Configure AWS Credentials"
        uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}
      - name: "📦 Organize External Libraries"
        run: |
          rm -rf \
            langchain/libs/partners/google-genai \
            langchain/libs/partners/google-vertexai
          mv langchain-google/libs/genai langchain/libs/partners/google-genai
          mv langchain-google/libs/vertexai langchain/libs/partners/google-vertexai
          mv langchain-aws/libs/aws langchain/libs/partners/aws

      - name: "🐍 Set up Python ${{ matrix.python-version }} + UV"
        uses: "./langchain/.github/actions/uv_setup"
        with:
          python-version: ${{ matrix.python-version }}

      - name: "📦 Install Dependencies"
        # Partner packages use [tool.uv.sources] in their pyproject.toml to resolve
        # langchain-core/langchain to local editable installs, so `uv sync` automatically
        # tests against the versions from the current branch (not published releases).
        #
        # External google/aws packages live in separate repos and don't declare
        # [tool.uv.sources], so `uv sync` pulls langchain-* from PyPI. Overlay
        # local editable installs after sync so integration tests exercise the
        # current branch's langchain code. Matches the pattern used by the
        # `test-dependents` job below for deepagents.
        run: |
          echo "Running scheduled tests, installing dependencies with uv..."
          cd langchain/${{ matrix.working-directory }}
          uv sync --group test --group test_integration

          case "${{ matrix.working-directory }}" in
            libs/partners/google-genai)
              uv pip install \
                -e $GITHUB_WORKSPACE/langchain/libs/core \
                -e $GITHUB_WORKSPACE/langchain/libs/standard-tests
              ;;
            libs/partners/google-vertexai)
              uv pip install \
                -e $GITHUB_WORKSPACE/langchain/libs/core \
                -e $GITHUB_WORKSPACE/langchain/libs/langchain_v1 \
                -e $GITHUB_WORKSPACE/langchain/libs/standard-tests
              ;;
            libs/partners/aws)
              uv pip install \
                -e $GITHUB_WORKSPACE/langchain/libs/core \
                -e $GITHUB_WORKSPACE/langchain/libs/langchain_v1 \
                -e $GITHUB_WORKSPACE/langchain/libs/langchain \
                -e $GITHUB_WORKSPACE/langchain/libs/standard-tests \
                -e $GITHUB_WORKSPACE/langchain/libs/partners/anthropic
              ;;
          esac

      - name: "🧾 Build LangSmith Metadata"
        # GHA expression values flow through intermediate env vars (injection
        # hardening) and jq -nc builds the JSON, so quotes/newlines in any
        # field can't corrupt the payload.
        env:
          GH_SHA: ${{ github.sha }}
          GH_RUN_ID: ${{ github.run_id }}
          GH_RUN_ATTEMPT: ${{ github.run_attempt }}
          GH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          GH_WORKFLOW: ${{ github.workflow }}
          GH_EVENT: ${{ github.event_name }}
          GH_REF: ${{ github.ref }}
          WORKING_DIRECTORY: ${{ matrix.working-directory }}
          PYTHON_VERSION: ${{ matrix.python-version }}
        run: |
          metadata=$(jq -nc \
            --arg github_sha "$GH_SHA" \
            --arg github_run_id "$GH_RUN_ID" \
            --arg github_run_attempt "$GH_RUN_ATTEMPT" \
            --arg github_run_url "$GH_RUN_URL" \
            --arg github_workflow "$GH_WORKFLOW" \
            --arg github_event "$GH_EVENT" \
            --arg github_ref "$GH_REF" \
            --arg working_directory "$WORKING_DIRECTORY" \
            --arg python_version "$PYTHON_VERSION" \
            '{github_sha: $github_sha, github_run_id: $github_run_id, github_run_attempt: $github_run_attempt, github_run_url: $github_run_url, github_workflow: $github_workflow, github_event: $github_event, github_ref: $github_ref, working_directory: $working_directory, python_version: $python_version}')
          echo "LANGSMITH_METADATA=$metadata" >> "$GITHUB_ENV"

      - name: "🚀 Run Integration Tests"
        # WARNING: All secrets below are available to every matrix job regardless of
        # which package is being tested. This is intentional for simplicity, but means
        # any test file could technically access any key. Only use for trusted code.
        env:
          LANGCHAIN_TESTS_USER_AGENT: ${{ secrets.LANGCHAIN_TESTS_USER_AGENT }}

          # Route traces to one project with GitHub run metadata so failures link back to the originating Actions run.
          LANGSMITH_TRACING: "true"
          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
          LANGSMITH_PROJECT: ${{ vars.LANGSMITH_PROJECT || 'scheduled-testing-py' }}
          LANGSMITH_TAGS: "github-actions,${{ matrix.working-directory }},python-${{ matrix.python-version }},sha-${{ github.sha }}"

          AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          ANTHROPIC_FILES_API_IMAGE_ID: ${{ secrets.ANTHROPIC_FILES_API_IMAGE_ID }}
          ANTHROPIC_FILES_API_PDF_ID: ${{ secrets.ANTHROPIC_FILES_API_PDF_ID }}
          ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
          ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
          ASTRA_DB_KEYSPACE: ${{ secrets.ASTRA_DB_KEYSPACE }}
          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
          AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          ES_URL: ${{ secrets.ES_URL }}
          ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
          ES_API_KEY: ${{ secrets.ES_API_KEY }}
          EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }}
          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
          NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
          PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}
          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
          UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
          WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
          WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
        run: |
          cd langchain/${{ matrix.working-directory }}
          make integration_tests

      - name: "🧹 Clean up External Libraries"
        # Clean up external libraries to avoid affecting the following git status check
        run: |
          rm -rf \
            langchain/libs/partners/google-genai \
            langchain/libs/partners/google-vertexai \
            langchain/libs/partners/aws

      - name: "🧹 Verify Clean Working Directory"
        working-directory: langchain
        run: |
          set -eu

          STATUS="$(git status)"
          echo "$STATUS"

          # grep will exit non-zero if the target message isn't found,
          # and `set -e` above will cause the step to fail.
          echo "$STATUS" | grep 'nothing to commit, working tree clean'

  # Test dependent packages against local packages to catch breaking changes
  test-dependents:
    # Defend against forks running scheduled jobs, but allow manual runs from forks
    if: github.repository_owner == 'langchain-ai' || github.event_name != 'schedule'

    name: "🐍 Python ${{ matrix.python-version }}: ${{ matrix.package.path }}"
    runs-on: ubuntu-latest
    needs: [compute-matrix]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        # deepagents requires Python >= 3.11, use bounded version from compute-matrix
        python-version: ${{ fromJSON(needs.compute-matrix.outputs.python-version-min-3-11) }}
        package:
          - name: deepagents
            repo: langchain-ai/deepagents
            path: libs/deepagents

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          path: langchain

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: ${{ matrix.package.repo }}
          path: ${{ matrix.package.name }}

      - name: "🐍 Set up Python ${{ matrix.python-version }} + UV"
        uses: "./langchain/.github/actions/uv_setup"
        with:
          python-version: ${{ matrix.python-version }}

      - name: "📦 Install ${{ matrix.package.name }} with Local"
        # Unlike partner packages (which use [tool.uv.sources] for local resolution),
        # external dependents live in separate repos and need explicit overrides to
        # test against the langchain versions from the current branch, as their
        # pyproject.toml files point to released versions.
        run: |
          cd ${{ matrix.package.name }}/${{ matrix.package.path }}

          # Install the package with test dependencies
          uv sync --group test

          # Override langchain packages with local versions
          uv pip install \
            -e $GITHUB_WORKSPACE/langchain/libs/core \
            -e $GITHUB_WORKSPACE/langchain/libs/langchain_v1

      # No API keys needed for now - deepagents `make test` only runs unit tests
      - name: "🚀 Run ${{ matrix.package.name }} Tests"
        run: |
          cd ${{ matrix.package.name }}/${{ matrix.package.path }}
          make test