Separate out langchain_core package (#13577)

Co-authored-by: Nuno Campos <nuno@boringbits.io>
Co-authored-by: Bagatur <baskaryan@gmail.com>
Co-authored-by: Erick Friis <erick@langchain.dev>

.github/workflows/_compile_integration_test.yml

@@ -7,6 +7,10 @@ on:
         required: true
         type: string
         description: "From which folder this pipeline executes"
+      langchain-core-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain core library folder"
 
 env:
   POETRY_VERSION: "1.6.1"
@@ -40,6 +44,14 @@ jobs:
         shell: bash
         run: poetry install --with=test_integration
 
+      - name: Install langchain core editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-core-location }}
+        env:
+          LANGCHAIN_CORE_LOCATION: ${{ inputs.langchain-core-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_CORE_LOCATION"
+
       - name: Check integration tests compile
         shell: bash
         run: poetry run pytest -m compile tests/integration_tests

.github/workflows/_lint.yml

@@ -11,6 +11,10 @@ on:
         required: false
         type: string
         description: "Relative path to the langchain library folder"
+      langchain-core-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain core library folder"
 
 env:
   POETRY_VERSION: "1.6.1"
@@ -76,7 +80,15 @@ jobs:
         env:
           LANGCHAIN_LOCATION: ${{ inputs.langchain-location }}
         run: |
-          pip install -e "$LANGCHAIN_LOCATION"
+          poetry run pip install -e "$LANGCHAIN_LOCATION"
+
+      - name: Install langchain core editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-core-location }}
+        env:
+          LANGCHAIN_CORE_LOCATION: ${{ inputs.langchain-core-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_CORE_LOCATION"
 
       - name: Get .mypy_cache to speed up mypy
         uses: actions/cache@v3

.github/workflows/_pydantic_compatibility.yml

@@ -7,6 +7,14 @@ on:
         required: true
         type: string
         description: "From which folder this pipeline executes"
+      langchain-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain library folder"
+      langchain-core-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain core library folder"
 
 env:
   POETRY_VERSION: "1.6.1"
@@ -40,6 +48,22 @@ jobs:
         shell: bash
         run: poetry install
 
+      - name: Install langchain editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-location }}
+        env:
+          LANGCHAIN_LOCATION: ${{ inputs.langchain-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_LOCATION"
+
+      - name: Install langchain core editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-core-location }}
+        env:
+          LANGCHAIN_CORE_LOCATION: ${{ inputs.langchain-core-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_CORE_LOCATION"
+
       - name: Install the opposite major version of pydantic
         # If normal tests use pydantic v1, here we'll use v2, and vice versa.
         shell: bash

.github/workflows/_test.yml

@@ -7,6 +7,14 @@ on:
         required: true
         type: string
         description: "From which folder this pipeline executes"
+      langchain-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain library folder"
+      langchain-core-location:
+        required: false
+        type: string
+        description: "Relative path to the langchain core library folder"
 
 env:
   POETRY_VERSION: "1.6.1"
@@ -40,9 +48,26 @@ jobs:
         shell: bash
         run: poetry install
 
+      - name: Install langchain editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-location }}
+        env:
+          LANGCHAIN_LOCATION: ${{ inputs.langchain-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_LOCATION"
+
+      - name: Install langchain core editable
+        working-directory: ${{ inputs.working-directory }}
+        if: ${{ inputs.langchain-core-location }}
+        env:
+          LANGCHAIN_CORE_LOCATION: ${{ inputs.langchain-core-location }}
+        run: |
+          poetry run pip install -e "$LANGCHAIN_CORE_LOCATION"
+
       - name: Run core tests
         shell: bash
-        run: make test
+        run: |
+          make test
 
       - name: Ensure the tests did not create any additional files
         shell: bash

.github/workflows/langchain_ci.yml

@@ -36,6 +36,7 @@ jobs:
       ./.github/workflows/_lint.yml
     with:
       working-directory: libs/langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   test:
@@ -43,6 +44,7 @@ jobs:
       ./.github/workflows/_test.yml
     with:
       working-directory: libs/langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   compile-integration-tests:
@@ -50,6 +52,7 @@ jobs:
       ./.github/workflows/_compile_integration_test.yml
     with:
       working-directory: libs/langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   pydantic-compatibility:
@@ -57,6 +60,7 @@ jobs:
       ./.github/workflows/_pydantic_compatibility.yml
     with:
       working-directory: libs/langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   extended-tests:
@@ -89,6 +93,11 @@ jobs:
           echo "Running extended tests, installing dependencies with poetry..."
           poetry install -E extended_testing
 
+      - name: Install langchain core editable
+        shell: bash
+        run: |
+          poetry run pip install -e ../core
+
       - name: Run extended tests
         run: make extended_tests
 

.github/workflows/langchain_core_ci.yml

@@ -0,0 +1,52 @@
+---
+name: libs/langchain core CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    paths:
+      - '.github/actions/poetry_setup/action.yml'
+      - '.github/tools/**'
+      - '.github/workflows/_lint.yml'
+      - '.github/workflows/_test.yml'
+      - '.github/workflows/_pydantic_compatibility.yml'
+      - '.github/workflows/langchain_core_ci.yml'
+      - 'libs/core/**'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: "libs/core"
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: libs/core
+    secrets: inherit
+
+  test:
+    uses:
+      ./.github/workflows/_test.yml
+    with:
+      working-directory: libs/core
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: libs/core
+    secrets: inherit

.github/workflows/langchain_core_release.yml

@@ -0,0 +1,13 @@
+---
+name: libs/core Release
+
+on:
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: libs/core
+    secrets: inherit

.github/workflows/langchain_experimental_ci.yml

@@ -36,6 +36,7 @@ jobs:
     with:
       working-directory: libs/experimental
       langchain-location: ../langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   test:
@@ -43,6 +44,8 @@ jobs:
       ./.github/workflows/_test.yml
     with:
       working-directory: libs/experimental
+      langchain-location: ../langchain
+      langchain-core-location: ../core
     secrets: inherit
 
   compile-integration-tests:
@@ -88,6 +91,7 @@ jobs:
 
           echo "Editably installing langchain outside of poetry, to avoid messing up lockfile..."
           poetry run pip install -e ../langchain
+          poetry run pip install -e ../core
 
       - name: Run tests
         run: make test