Merge branch 'master' into array_data_type

Author: rgupta2508

.devcontainer/docker-compose.yaml

@@ -5,10 +5,10 @@ services:
       dockerfile: libs/langchain/dev.Dockerfile
       context: ..
     volumes:
-   # Update this to wherever you want VS Code to mount the folder of your project
+      # Update this to wherever you want VS Code to mount the folder of your project
       - ..:/workspaces/langchain:cached
     networks:
-      - langchain-network 
+      - langchain-network
   #   environment:
   #     MONGO_ROOT_USERNAME: root
   #     MONGO_ROOT_PASSWORD: example123
@@ -28,5 +28,3 @@ services:
 networks:
   langchain-network:
     driver: bridge
-    
-    

.github/scripts/check_diff.py

@@ -1,7 +1,6 @@
 import glob
 import json
 import os
-import re
 import sys
 import tomllib
 from collections import defaultdict
@@ -86,6 +85,11 @@ def add_dependents(dirs_to_eval: Set[str], dependents: dict) -> List[str]:
 
 
 def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
+    if dir_ == "libs/core":
+        return [
+            {"working-directory": dir_, "python-version": f"3.{v}"}
+            for v in range(8, 13)
+        ]
     min_python = "3.8"
     max_python = "3.12"
 
@@ -95,6 +99,15 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
         # declare deps in funny way
         max_python = "3.11"
 
+    if dir_ in ["libs/community", "libs/langchain"] and job == "extended-tests":
+        # community extended test resolution in 3.12 is slow
+        # even in uv
+        max_python = "3.11"
+
+    if dir_ == "libs/community" and job == "compile-integration-tests":
+        # community integration deps are slow in 3.12
+        max_python = "3.11"
+
     return [
         {"working-directory": dir_, "python-version": min_python},
         {"working-directory": dir_, "python-version": max_python},

.github/scripts/check_prerelease_dependencies.py

@@ -0,0 +1,35 @@
+import sys
+import tomllib
+
+if __name__ == "__main__":
+    # Get the TOML file path from the command line argument
+    toml_file = sys.argv[1]
+
+    # read toml file
+    with open(toml_file, "rb") as file:
+        toml_data = tomllib.load(file)
+
+    # see if we're releasing an rc
+    version = toml_data["tool"]["poetry"]["version"]
+    releasing_rc = "rc" in version
+
+    # if not, iterate through dependencies and make sure none allow prereleases
+    if not releasing_rc:
+        dependencies = toml_data["tool"]["poetry"]["dependencies"]
+        for lib in dependencies:
+            dep_version = dependencies[lib]
+            dep_version_string = (
+                dep_version["version"] if isinstance(dep_version, dict) else dep_version
+            )
+
+            if "rc" in dep_version_string:
+                raise ValueError(
+                    f"Dependency {lib} has a prerelease version. Please remove this."
+                )
+
+            if isinstance(dep_version, dict) and dep_version.get(
+                "allow-prereleases", False
+            ):
+                raise ValueError(
+                    f"Dependency {lib} has allow-prereleases set to true. Please remove this."
+                )

.github/scripts/get_min_versions.py

@@ -1,6 +1,11 @@
 import sys
 
-import tomllib
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    # for python 3.10 and below, which doesnt have stdlib tomllib
+    import tomli as tomllib
+
 from packaging.version import parse as parse_version
 import re
 
@@ -12,6 +17,8 @@
     "SQLAlchemy",
 ]
 
+SKIP_IF_PULL_REQUEST = ["langchain-core"]
+
 
 def get_min_version(version: str) -> str:
     # base regex for x.x.x with cases for rc/post/etc
@@ -38,7 +45,7 @@ def get_min_version(version: str) -> str:
     raise ValueError(f"Unrecognized version format: {version}")
 
 
-def get_min_version_from_toml(toml_path: str):
+def get_min_version_from_toml(toml_path: str, versions_for: str):
     # Parse the TOML file
     with open(toml_path, "rb") as file:
         toml_data = tomllib.load(file)
@@ -51,6 +58,10 @@ def get_min_version_from_toml(toml_path: str):
 
     # Iterate over the libs in MIN_VERSION_LIBS
     for lib in MIN_VERSION_LIBS:
+        if versions_for == "pull_request" and lib in SKIP_IF_PULL_REQUEST:
+            # some libs only get checked on release because of simultaneous
+            # changes
+            continue
         # Check if the lib is present in the dependencies
         if lib in dependencies:
             # Get the version string
@@ -71,8 +82,10 @@ def get_min_version_from_toml(toml_path: str):
 if __name__ == "__main__":
     # Get the TOML file path from the command line argument
     toml_file = sys.argv[1]
+    versions_for = sys.argv[2]
+    assert versions_for in ["release", "pull_request"]
 
     # Call the function to get the minimum versions
-    min_versions = get_min_version_from_toml(toml_file)
+    min_versions = get_min_version_from_toml(toml_file, versions_for)
 
     print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))

.github/workflows/_release.yml

@@ -189,7 +189,7 @@ jobs:
             --extra-index-url https://test.pypi.org/simple/ \
             "$PKG_NAME==$VERSION" || \
           ( \
-            sleep 5 && \
+            sleep 15 && \
             poetry run pip install \
               --extra-index-url https://test.pypi.org/simple/ \
               "$PKG_NAME==$VERSION" \
@@ -221,12 +221,17 @@ jobs:
         run: make tests
         working-directory: ${{ inputs.working-directory }}
 
+      - name: Check for prerelease versions
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry run python $GITHUB_WORKSPACE/.github/scripts/check_prerelease_dependencies.py pyproject.toml
+
       - name: Get minimum versions
         working-directory: ${{ inputs.working-directory }}
         id: min-version
         run: |
           poetry run pip install packaging
-          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
+          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml release)"
           echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
           echo "min-versions=$min_versions"
 
@@ -285,6 +290,7 @@ jobs:
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
           FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+          UNSTRUCTURED_API_KEY: ${{ secrets.UNSTRUCTURED_API_KEY }}
         run: make integration_tests
         working-directory: ${{ inputs.working-directory }}
 

.github/workflows/_test.yml

@@ -65,3 +65,22 @@ jobs:
           # grep will exit non-zero if the target message isn't found,
           # and `set -e` above will cause the step to fail.
           echo "$STATUS" | grep 'nothing to commit, working tree clean'
+          
+      - name: Get minimum versions
+        working-directory: ${{ inputs.working-directory }}
+        id: min-version
+        run: |
+          poetry run pip install packaging tomli
+          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml pull_request)"
+          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
+          echo "min-versions=$min_versions"
+
+# Temporarily disabled until we can get the minimum versions working
+#      - name: Run unit tests with minimum dependency versions
+#        if: ${{ steps.min-version.outputs.min-versions != '' }}
+#        env:
+#          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
+#        run: |
+#          poetry run pip install --force-reinstall $MIN_VERSIONS --editable .
+#          make tests
+#        working-directory: ${{ inputs.working-directory }}

MIGRATE.md

@@ -52,7 +52,7 @@ Now:
 
 `from langchain_experimental.sql import SQLDatabaseChain`
 
-Alternatively, if you are just interested in using the query generation part of the SQL chain, you can check out [`create_sql_query_chain`](https://github.com/langchain-ai/langchain/blob/master/docs/extras/use_cases/tabular/sql_query.ipynb)
+Alternatively, if you are just interested in using the query generation part of the SQL chain, you can check out this [`SQL question-answering tutorial`](https://python.langchain.com/v0.2/docs/tutorials/sql_qa/#convert-question-to-sql-query)
 
 `from langchain.chains import create_sql_query_chain`
 

README.md

@@ -7,7 +7,6 @@
 [![PyPI - License](https://img.shields.io/pypi/l/langchain-core?style=flat-square)](https://opensource.org/licenses/MIT)
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-core?style=flat-square)](https://pypistats.org/packages/langchain-core)
 [![GitHub star chart](https://img.shields.io/github/stars/langchain-ai/langchain?style=flat-square)](https://star-history.com/#langchain-ai/langchain)
-[![Dependency Status](https://img.shields.io/librariesio/github/langchain-ai/langchain?style=flat-square)](https://libraries.io/github/langchain-ai/langchain)
 [![Open Issues](https://img.shields.io/github/issues-raw/langchain-ai/langchain?style=flat-square)](https://github.com/langchain-ai/langchain/issues)
 [![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode&style=flat-square)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/langchain-ai/langchain)
 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/langchain-ai/langchain)

cookbook/Multi_modal_RAG.ipynb

@@ -64,7 +64,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! pip install -U langchain openai chromadb langchain-experimental # (newest versions required for multi-modal)"
+    "! pip install -U langchain openai langchain-chroma langchain-experimental # (newest versions required for multi-modal)"
    ]
   },
   {
@@ -355,7 +355,7 @@
     "\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_chroma import Chroma\n",
     "from langchain_core.documents import Document\n",
     "from langchain_openai import OpenAIEmbeddings\n",
     "\n",

cookbook/Multi_modal_RAG_google.ipynb

@@ -37,7 +37,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install -U --quiet langchain langchain_community openai chromadb langchain-experimental\n",
+    "%pip install -U --quiet langchain langchain-chroma langchain-community openai langchain-experimental\n",
     "%pip install --quiet \"unstructured[all-docs]\" pypdf pillow pydantic lxml pillow matplotlib chromadb tiktoken"
    ]
   },
@@ -344,8 +344,8 @@
     "\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
+    "from langchain_chroma import Chroma\n",
     "from langchain_community.embeddings import VertexAIEmbeddings\n",
-    "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.documents import Document\n",
     "\n",
     "\n",