Add CUDA version compatibility check

Andy-Jost · Andy-Jost · commit 1962e351a233 · 2025-12-18T11:08:03.000-08:00
Warn when cuda-bindings was compiled against a newer CUDA major version
than the installed driver supports. This helps users understand why
certain features may not work correctly.

The check runs once after cuInit and can be suppressed via
CUDA_PYTHON_DISABLE_VERSION_CHECK=1.
diff --git a/cuda_bindings/docs/source/environment_variables.rst b/cuda_bindings/docs/source/environment_variables.rst
@@ -9,6 +9,8 @@ Runtime Environment Variables
 
 - ``CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM`` : When set to 1, the default stream is the per-thread default stream. When set to 0, the default stream is the legacy default stream. This defaults to 0, for the legacy default stream. See `Stream Synchronization Behavior <https://docs.nvidia.com/cuda/cuda-runtime-api/stream-sync-behavior.html>`_ for an explanation of the legacy and per-thread default streams.
 
+- ``CUDA_PYTHON_DISABLE_VERSION_CHECK`` : When set to 1, suppresses the warning that is issued when ``cuda.core`` detects that ``cuda-bindings`` was compiled against a newer CUDA major version than the installed driver supports. This warning helps identify version mismatches that may cause features to not work correctly.
+
 
 Build-Time Environment Variables
 --------------------------------
diff --git a/cuda_core/cuda/core/_device.pyx b/cuda_core/cuda/core/_device.pyx
@@ -17,6 +17,7 @@ from cuda.core._graph import GraphBuilder
 from cuda.core._stream import IsStreamT, Stream, StreamOptions
 from cuda.core._utils.clear_error_support import assert_type
 from cuda.core._utils.cuda_utils import (
+    check_cuda_version_compatibility,
     ComputeCapability,
     CUDAError,
     driver,
@@ -963,6 +964,8 @@ class Device:
             with _lock, nogil:
                 HANDLE_RETURN(cydriver.cuInit(0))
                 _is_cuInit = True
+            # Check version compatibility after CUDA is initialized
+            check_cuda_version_compatibility()
 
         # important: creating a Device instance does not initialize the GPU!
         cdef cydriver.CUdevice dev
diff --git a/cuda_core/cuda/core/_utils/cuda_utils.pyx b/cuda_core/cuda/core/_utils/cuda_utils.pyx
@@ -6,6 +6,7 @@ import functools
 from functools import partial
 import importlib.metadata
 import multiprocessing
+import os
 import platform
 import warnings
 from collections import namedtuple
@@ -288,6 +289,67 @@ class Transaction:
         self._stack.pop_all()
 
 
+# Track whether we've already checked version compatibility
+_version_compatibility_checked = False
+
+
+def reset_version_compatibility_check():
+    """Reset the version compatibility check flag for testing purposes.
+
+    This function is intended for use in tests to allow multiple test runs
+    to check the warning behavior.
+    """
+    global _version_compatibility_checked
+    _version_compatibility_checked = False
+
+
+def check_cuda_version_compatibility():
+    """Check if the CUDA driver version is compatible with cuda-bindings compile-time version.
+
+    This function compares the CUDA version that cuda-bindings was compiled against
+    with the CUDA version supported by the installed driver. If the compile-time
+    major version is greater than the driver's major version, a warning is issued.
+
+    The warning can be suppressed by setting the environment variable
+    CUDA_PYTHON_DISABLE_VERSION_CHECK=1.
+    """
+    global _version_compatibility_checked
+    if _version_compatibility_checked:
+        return
+    _version_compatibility_checked = True
+
+    # Allow users to suppress the warning
+    if os.environ.get("CUDA_PYTHON_DISABLE_VERSION_CHECK"):
+        return
+
+    # Get compile-time CUDA version from cuda-bindings
+    try:
+        compile_version = driver.CUDA_VERSION  # e.g., 13010
+    except AttributeError:
+        # Older cuda-bindings may not expose CUDA_VERSION
+        return
+
+    # Get runtime driver version
+    err, runtime_version = driver.cuDriverGetVersion()
+    if err != driver.CUresult.CUDA_SUCCESS:
+        return  # Can't check, skip silently
+
+    compile_major = compile_version // 1000
+    runtime_major = runtime_version // 1000
+
+    if compile_major > runtime_major:
+        compile_minor = (compile_version % 1000) // 10
+        runtime_minor = (runtime_version % 1000) // 10
+        warnings.warn(
+            f"cuda-python was built against CUDA {compile_major}.{compile_minor}, "
+            f"but the installed driver only supports CUDA {runtime_major}.{runtime_minor}. "
+            f"Some features may not work correctly. Consider updating your NVIDIA driver. "
+            f"Set CUDA_PYTHON_DISABLE_VERSION_CHECK=1 to suppress this warning.",
+            UserWarning,
+            stacklevel=3,
+        )
+
+
 # Track whether we've already warned about fork method
 _fork_warning_checked = False
 
diff --git a/cuda_core/tests/test_cuda_utils.py b/cuda_core/tests/test_cuda_utils.py
@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE
 
+import os
+import warnings
+from unittest import mock
+
 import pytest
 from cuda.bindings import driver, runtime
 from cuda.core._utils import cuda_utils
@@ -75,3 +79,108 @@ def test_check_runtime_error():
                 assert enum_name in msg
     # Smoke test: We don't want most to be unexpected.
     assert num_unexpected < len(driver.CUresult) * 0.5
+
+
+class TestVersionCompatibilityCheck:
+    """Tests for check_cuda_version_compatibility function."""
+
+    def setup_method(self):
+        """Reset the version compatibility check flag before each test."""
+        cuda_utils.reset_version_compatibility_check()
+
+    def teardown_method(self):
+        """Reset the version compatibility check flag after each test."""
+        cuda_utils.reset_version_compatibility_check()
+
+    def test_no_warning_when_driver_newer(self):
+        """No warning should be issued when driver version >= compile version."""
+        # Mock compile version 12.9 and driver version 13.0
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 12090),
+            mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 13000)),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            assert len(w) == 0
+
+    def test_no_warning_when_same_major_version(self):
+        """No warning should be issued when major versions match."""
+        # Mock compile version 12.9 and driver version 12.8
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 12090),
+            mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            assert len(w) == 0
+
+    def test_warning_when_compile_major_newer(self):
+        """Warning should be issued when compile major version > driver major version."""
+        # Mock compile version 13.0 and driver version 12.8
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 13000),
+            mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            assert len(w) == 1
+            assert issubclass(w[0].category, UserWarning)
+            assert "cuda-python was built against CUDA 13.0" in str(w[0].message)
+            assert "driver only supports CUDA 12.8" in str(w[0].message)
+
+    def test_warning_only_issued_once(self):
+        """Warning should only be issued once per process."""
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 13000),
+            mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            cuda_utils.check_cuda_version_compatibility()
+            cuda_utils.check_cuda_version_compatibility()
+            # Only one warning despite multiple calls
+            assert len(w) == 1
+
+    def test_warning_suppressed_by_env_var(self):
+        """Warning should be suppressed when CUDA_PYTHON_DISABLE_VERSION_CHECK is set."""
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 13000),
+            mock.patch.object(driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_SUCCESS, 12080)),
+            mock.patch.dict(os.environ, {"CUDA_PYTHON_DISABLE_VERSION_CHECK": "1"}),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            assert len(w) == 0
+
+    def test_silent_when_driver_version_fails(self):
+        """Should silently skip if cuDriverGetVersion fails."""
+        with (
+            mock.patch.object(driver, "CUDA_VERSION", 13000),
+            mock.patch.object(
+                driver, "cuDriverGetVersion", return_value=(driver.CUresult.CUDA_ERROR_NOT_INITIALIZED, 0)
+            ),
+            warnings.catch_warnings(record=True) as w,
+        ):
+            warnings.simplefilter("always")
+            cuda_utils.check_cuda_version_compatibility()
+            assert len(w) == 0
+
+    def test_silent_when_cuda_version_not_available(self):
+        """Should silently skip if CUDA_VERSION attribute is not available."""
+        # Simulate older cuda-bindings without CUDA_VERSION
+        with mock.patch.object(driver, "CUDA_VERSION", None):
+            # Make accessing CUDA_VERSION raise AttributeError
+            original = driver.CUDA_VERSION
+            del driver.CUDA_VERSION
+            try:
+                with warnings.catch_warnings(record=True) as w:
+                    warnings.simplefilter("always")
+                    cuda_utils.check_cuda_version_compatibility()
+                    assert len(w) == 0
+            finally:
+                driver.CUDA_VERSION = original
