From 22da9f5f3f9fef24c5c75072b678b8a2f654b173 Mon Sep 17 00:00:00 2001 From: ccurme Date: Tue, 16 Apr 2024 16:49:46 -0400 Subject: [PATCH] update scheduled tests (#20526) repurpose scheduled tests to test over provider packages --- .github/workflows/_integration_test.yml | 1 + .github/workflows/scheduled_test.yml | 42 +++++++++---------- libs/langchain/Makefile | 3 -- .../embeddings/test_azure.py | 2 +- libs/partners/together/Makefile | 5 +-- 5 files changed, 23 insertions(+), 30 deletions(-) diff --git a/.github/workflows/_integration_test.yml b/.github/workflows/_integration_test.yml index 3d4019bd125..58f27c97440 100644 --- a/.github/workflows/_integration_test.yml +++ b/.github/workflows/_integration_test.yml @@ -58,6 +58,7 @@ jobs: MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }} GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} diff --git a/.github/workflows/scheduled_test.yml b/.github/workflows/scheduled_test.yml index a5bae539102..1b6522385de 100644 --- a/.github/workflows/scheduled_test.yml +++ b/.github/workflows/scheduled_test.yml @@ -10,19 +10,21 @@ env: jobs: build: - defaults: - run: - working-directory: libs/langchain runs-on: ubuntu-latest - environment: Scheduled testing strategy: matrix: python-version: - "3.8" - - "3.9" - - "3.10" - "3.11" - name: Python ${{ matrix.python-version }} + working-directory: + - "libs/partners/openai" + - "libs/partners/anthropic" + # - "libs/partners/ai21" # standard-tests broken + - "libs/partners/fireworks" + # - "libs/partners/groq" # rate-limited + - "libs/partners/mistralai" + # - "libs/partners/together" # rate-limited + name: Python ${{ matrix.python-version }} - ${{ matrix.working-directory }} steps: - uses: actions/checkout@v4 @@ -31,7 +33,7 @@ jobs: with: python-version: ${{ matrix.python-version }} poetry-version: ${{ env.POETRY_VERSION }} - working-directory: libs/langchain + working-directory: ${{ matrix.working-directory }} cache-key: scheduled - name: 'Authenticate to Google Cloud' @@ -40,26 +42,15 @@ jobs: with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - name: Install dependencies - working-directory: libs/langchain + working-directory: ${{ matrix.working-directory }} shell: bash run: | echo "Running scheduled tests, installing dependencies with poetry..." poetry install --with=test_integration,test - - name: Install deps outside pyproject - if: ${{ startsWith(inputs.working-directory, 'libs/community/') }} - shell: bash - run: poetry run pip install "boto3<2" "google-cloud-aiplatform<2" - - - name: Run tests + - name: Run integration tests + working-directory: ${{ matrix.working-directory }} shell: bash env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -70,11 +61,16 @@ jobs: AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }} AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }} AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} + AI21_API_KEY: ${{ secrets.AI21_API_KEY }} FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} run: | - make scheduled_tests + make integration_test - name: Ensure the tests did not create any additional files + working-directory: ${{ matrix.working-directory }} shell: bash run: | set -eu diff --git a/libs/langchain/Makefile b/libs/langchain/Makefile index 119e15422f8..6dc275d6f47 100644 --- a/libs/langchain/Makefile +++ b/libs/langchain/Makefile @@ -33,9 +33,6 @@ test_watch_extended: integration_tests: poetry run pytest tests/integration_tests -scheduled_tests: - poetry run pytest -m scheduled tests/integration_tests - docker_tests: docker build -t my-langchain-image:test . docker run --rm my-langchain-image:test diff --git a/libs/partners/openai/tests/integration_tests/embeddings/test_azure.py b/libs/partners/openai/tests/integration_tests/embeddings/test_azure.py index 6f697c9a3b0..ad01bc1a61d 100644 --- a/libs/partners/openai/tests/integration_tests/embeddings/test_azure.py +++ b/libs/partners/openai/tests/integration_tests/embeddings/test_azure.py @@ -117,7 +117,7 @@ def test_azure_openai_embedding_with_empty_string() -> None: .data[0] .embedding ) - assert np.allclose(output[0], expected_output) + assert np.allclose(output[0], expected_output, atol=0.0001) assert len(output[1]) == 1536 diff --git a/libs/partners/together/Makefile b/libs/partners/together/Makefile index f231cd432fa..8a8fb9add2e 100644 --- a/libs/partners/together/Makefile +++ b/libs/partners/together/Makefile @@ -6,10 +6,9 @@ all: help # Define a variable for the test file path. TEST_FILE ?= tests/unit_tests/ -test: - poetry run pytest $(TEST_FILE) +integration_test integration_tests: TEST_FILE=tests/integration_tests/ -tests: +test tests integration_test integration_tests: poetry run pytest $(TEST_FILE)