Merge remote-tracking branch 'upstream/master' into xds_python_protos_move

markdroth · markdroth · commit deac934aa69c · 2025-12-19T01:39:09.000Z
diff --git a/.pylintrc b/.pylintrc
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
 	src/python/grpcio/grpc/framework/common,
 	src/python/grpcio/grpc/framework/foundation,
 	src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,
 
 extension-pkg-whitelist=grpc._cython.cygrpc
 
diff --git a/.pylintrc-examples b/.pylintrc-examples
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -10,6 +11,9 @@ ignore=
 	src/python/grpcio/grpc/framework/common,
 	src/python/grpcio/grpc/framework/foundation,
 	src/python/grpcio/grpc/framework/interfaces,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,
 
 [VARIABLES]
 
diff --git a/.pylintrc-tests b/.pylintrc-tests
@@ -1,4 +1,5 @@
 [MAIN]
+py-version = 3.9
 load-plugins=
 	pylint.extensions.no_self_use,
 	pylint.extensions.comparison_placement,
@@ -9,6 +10,9 @@ ignore=
 	src/python/grpcio_tests/tests/unit/framework,
 	src/python/grpcio_tests/tests/unit/framework/common,
 	src/python/grpcio_tests/tests/unit/framework/foundation,
+ignore-patterns=
+	.*pb2\.py,
+	.*pb2_grpc\.py,
 
 [VARIABLES]
 
diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl
@@ -63,7 +63,7 @@ def generate_cc_impl(ctx):
     includes = [
         f
         for src in ctx.attr.srcs
-        for f in src[ProtoInfo].transitive_imports.to_list()
+        for f in src[ProtoInfo].transitive_sources.to_list()
     ]
     outs = []
     proto_root = get_proto_root(
diff --git a/bazel/generate_objc.bzl b/bazel/generate_objc.bzl
@@ -40,7 +40,7 @@ def _generate_objc_impl(ctx):
     protos = [
         f
         for src in ctx.attr.deps
-        for f in src[ProtoInfo].transitive_imports.to_list()
+        for f in src[ProtoInfo].transitive_sources.to_list()
     ]
 
     target_package = _join_directories([ctx.label.workspace_root, ctx.label.package])
diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl
@@ -233,7 +233,7 @@ def includes_from_deps(deps):
     return [
         file
         for src in deps
-        for file in src[ProtoInfo].transitive_imports.to_list()
+        for file in src[ProtoInfo].transitive_sources.to_list()
     ]
 
 def get_proto_arguments(protos, genfiles_dir_path):
diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl
@@ -62,7 +62,7 @@ def _gen_py_aspect_impl(target, context):
     for p in target[ProtoInfo].direct_sources:
         protos.append(get_staged_proto_file(target.label, context, p))
 
-    includes = depset(direct = protos, transitive = [target[ProtoInfo].transitive_imports])
+    includes = depset(direct = protos, transitive = [target[ProtoInfo].transitive_sources])
     out_files = (declare_out_files(protos, context, _GENERATED_PROTO_FORMAT) +
                  declare_out_files(protos, context, _GENERATED_PROTO_STUB_FORMAT))
     generated_py_srcs = out_files
diff --git a/grpc-style-config.toml b/grpc-style-config.toml
@@ -4,6 +4,8 @@ target-version = [
   "py39",
   "py310",
   "py311",
+  "py312",
+  "py313",
 ]
 extend-exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
@@ -29,6 +31,9 @@ extend-exclude = '''
 
 [tool.isort]
 profile = "black"
+py_version = 39
+follow_links = false
+
 line_length = 80
 src_paths = [
     "examples/python/data_transmission",
@@ -48,6 +53,7 @@ skip_glob = [
     "*pb2*.pyi",
     "**/site-packages/**/*",
     "py_xds_protos/*",
+    "**/.venv-*",
 ]
 single_line_exclusions = ["typing"]
 force_single_line = true
diff --git a/requirements.bazel.lock b/requirements.bazel.lock
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --allow-unsafe --output-file=requirements.bazel.lock requirements.bazel.txt
@@ -26,13 +26,13 @@ deprecated==1.2.18
     # via
     #   opentelemetry-api
     #   opentelemetry-semantic-conventions
-gevent==24.2.1
+gevent==25.9.1
     # via -r requirements.bazel.txt
 google-auth==2.38.0
     # via -r requirements.bazel.txt
 googleapis-common-protos==1.66.0
     # via -r requirements.bazel.txt
-greenlet==3.1.1
+greenlet==3.2.4
     # via gevent
 hyperlink==21.0.0
     # via twisted
@@ -77,7 +77,7 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pyyaml==6.0.2
+pyyaml==6.0.3
     # via -r requirements.bazel.txt
 requests==2.32.3
     # via
@@ -89,7 +89,7 @@ tomli==2.2.1
     # via incremental
 twisted==24.11.0
     # via -r requirements.bazel.txt
-typeguard==4.2.1
+typeguard==4.4.2
     # via -r requirements.bazel.txt
 typing-extensions==4.12.2
     # via
@@ -115,8 +115,7 @@ zope-interface==7.2
     #   twisted
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==77.0.1 ; python_version > "3.8" # Manually added for python > 3.8, until Bazel RBE jobs support Python >= 3.9 (Tech debt: b/427881645)
-setuptools==75.3.2 ; python_version <= "3.8"
+setuptools==77.0.1
     # via
     #   -r requirements.bazel.txt
     #   incremental
diff --git a/requirements.bazel.txt b/requirements.bazel.txt
@@ -16,7 +16,7 @@
 # Instructions for generation follow:
 # Use the oldest supported version of Python
 #
-#   $ docker run -it --rm -v $(pwd):/grpc python:3.8 /bin/bash
+#   $ docker run -it --rm -v $(pwd):/grpc python:3.9 /bin/bash
 #   # cd grpc
 #   # pip install pip-tools
 #   # pip-compile --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
@@ -26,17 +26,21 @@
 # When installing a new package, sometimes it's desirable to avoid unnecessary
 # package upgrades. In such case, consider the following flow instead.
 #
-#   $ python3.8 -m venv .venv-bazel
+#   $ python3.9 -m venv .venv-bazel
 #   $ source .venv-bazel/bin/activate
 #   $ pip install pip-tools
-#   $ pip install -r requirements.bazel.lock # make the change to version in requirements.bazel.txt after this
+#   $ pip install -r requirements.bazel.lock
+#
+#   Add the new package to requirements.bazel.txt
 #   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock
-#   $ deactivate
+#
+#   To update a single package, change its version spec requirements.bazel.txt
+#   $ pip-compile --no-upgrade --allow-unsafe requirements.bazel.txt -o requirements.bazel.lock -P package_name
 
 absl-py
 certifi
 chardet
-gevent
+gevent~=25.8
 google-auth
 googleapis-common-protos
 idna
@@ -47,11 +51,8 @@ opentelemetry-sdk
 protobuf
 pyyaml  # for DNS test
 requests
-setuptools>=77.0.1; python_version > "3.8"
-setuptools>=75.3.2; python_version <= "3.8"
-# Currently our CI uses Python < 3.8, hence <4.4.1 was added
-# TODO(asheshvidyut): remove the <4.4.1, when CI uses python >= 3.9
-typeguard~=4.2.0,<4.4.1
+setuptools>=77.0.1
+typeguard~=4.2
 typing-extensions==4.12.2
 twisted  # for DNS test
 urllib3
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/aio/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/aio/completion_queue.pyx.pxi
@@ -131,14 +131,20 @@ cdef class PollerCompletionQueue(BaseCompletionQueue):
         for loop in self._loops:
             self._loops.get(loop).close()
 
+        # Close the read socket to prevent the `_poller_thread` from blocking on a `write` syscall
+        # when the Unix-domain socket buffer is full. Once the loops above are closed, the read
+        # socket is no longer being read, so close it to avoid `write` syscall hangs.
+        #
+        # See `sock_alloc_send_pskb` for more details about these `write` syscall hangs.
+        self._read_socket.close()
+
         # TODO(https://github.com/grpc/grpc/issues/22365) perform graceful shutdown
         grpc_completion_queue_shutdown(self._cq)
         while not self._shutdown:
             self._poller_thread.join(timeout=_POLL_AWAKE_INTERVAL_S)
         grpc_completion_queue_destroy(self._cq)
 
-        # Clean up socket resources
-        self._read_socket.close()
+        # Clean up the write socket
         self._write_socket.close()
 
     def _handle_events(self, object context_loop):
diff --git a/tools/distrib/black_code.sh b/tools/distrib/black_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+set -eux
 
 ACTION="${1:-}"
 [[ $ACTION == '' ]] || [[ $ACTION == '--diff' ]] || [[ $ACTION == '--check' ]]
@@ -29,9 +29,12 @@ DIRS=(
     'setup.py'
 )
 
-VIRTUALENV=venv_black_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-black"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
-python3 -m pip install black==25.1.0
-python3 -m black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"
+pip install black==25.1.0
+pip list
+
+exec black --config=grpc-style-config.toml $ACTION "${DIRS[@]}"
diff --git a/tools/distrib/check_pytype.sh b/tools/distrib/check_pytype.sh
@@ -1,4 +1,4 @@
-#! /bin/bash -ex
+#!/usr/bin/env bash
 # Copyright 2025 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VIRTUALENV=.venv_check_pytype
-python3.11 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+set -eux
+
+VIRTUALENV=".venv-ci-pytype"
+python3.11 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
 pip install pytype==2024.10.11
+pip list
+
 pytype --output=~/.cache/pytype --config=grpc-style-config.toml
diff --git a/tools/distrib/isort_code.sh b/tools/distrib/isort_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2021 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
+set -eux
 
 ACTION=${1:---overwrite-in-place}
 [[ $ACTION == '--overwrite-in-place' ]] || [[ $ACTION == '--diff' ]]
@@ -33,9 +33,12 @@ DIRS=(
     'setup.py'
 )
 
-VIRTUALENV=venv_isort_code
-python3 -m virtualenv $VIRTUALENV
-source $VIRTUALENV/bin/activate
+VIRTUALENV=".venv-ci-isort"
+python3 -m virtualenv "${VIRTUALENV}"
+source "${VIRTUALENV}/bin/activate"
+python -VV
 
-python3 -m pip install isort==5.9.2
-python3 -m isort $ACTION --settings-path=grpc-style-config.toml --dont-follow-links "${DIRS[@]}"
+pip install isort==6.0.1
+pip list
+
+isort $ACTION --settings-path=grpc-style-config.toml "${DIRS[@]}"
diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Copyright 2017 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-# NOTE(rbellevi): We ignore generated code.
-IGNORE_PATTERNS=--ignore-patterns='.*pb2\.py,.*pb2_grpc\.py'
+set -eux
 
 # change to root directory
 cd "$(dirname "$0")/../.."
@@ -37,34 +34,25 @@ TEST_DIRS=(
     'src/python/grpcio_tests/tests_gevent'
 )
 
-VIRTUALENV=".venv-pylint"
-
-# When running locally, allow to reuse existing venv.
-if [[ ! -d "${VIRTUALENV}" ]]; then
-  python3.11 -m virtualenv "${VIRTUALENV}"
-fi
-
+VIRTUALENV=".venv-ci-pylint"
+python3.11 -m virtualenv "${VIRTUALENV}"
 source "${VIRTUALENV}/bin/activate"
-python3 --version
+python -VV
 
 # TODO(https://github.com/grpc/grpc/issues/23394): Update Pylint.
-pip install --upgrade astroid==2.15.8 \
-  pylint==2.17.7 \
-  "isort~=5.11"
+pip install --upgrade astroid==2.15.8 pylint==2.17.7 "isort~=5.11"
+pip list
 
 EXIT=0
 for dir in "${DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc -rn "$dir" ${IGNORE_PATTERNS}  || EXIT=1
+  pylint --rcfile=.pylintrc -rn "$dir" || EXIT=1
 done
 
 for dir in "${TEST_DIRS[@]}"; do
-  python3 -m pylint --rcfile=.pylintrc-tests -rn "$dir" ${IGNORE_PATTERNS} || EXIT=1
+  pylint --rcfile=.pylintrc-tests -rn "$dir" || EXIT=1
 done
 
-find examples/python \
-  -iname "*.py" \
-  -not -name "*_pb2.py" \
-  -not -name "*_pb2_grpc.py" \
-  | xargs python3 -m pylint --rcfile=.pylintrc-examples -rn ${IGNORE_PATTERNS}
+find examples/python -iname "*.py" -not -name "*_pb2*.py" \
+  | xargs pylint --rcfile=.pylintrc-examples -rn
 
 exit $EXIT
diff --git a/tools/distrib/ruff_code.sh b/tools/distrib/ruff_code.sh
diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc

Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ def generate_cc_impl(ctx):`
`63`	`63`	`includes = [`
`64`	`64`	`f`
`65`	`65`	`for src in ctx.attr.srcs`
`66`		`- for f in src[ProtoInfo].transitive_imports.to_list()`
	`66`	`+ for f in src[ProtoInfo].transitive_sources.to_list()`
`67`	`67`	`]`
`68`	`68`	`outs = []`
`69`	`69`	`proto_root = get_proto_root(`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ def _generate_objc_impl(ctx):`
`40`	`40`	`protos = [`
`41`	`41`	`f`
`42`	`42`	`for src in ctx.attr.deps`
`43`		`- for f in src[ProtoInfo].transitive_imports.to_list()`
	`43`	`+ for f in src[ProtoInfo].transitive_sources.to_list()`
`44`	`44`	`]`
`45`	`45`
`46`	`46`	`target_package = _join_directories([ctx.label.workspace_root, ctx.label.package])`
Original file line number	Diff line number	Diff line change
`@@ -233,7 +233,7 @@ def includes_from_deps(deps):`
`233`	`233`	`return [`
`234`	`234`	`file`
`235`	`235`	`for src in deps`
`236`		`- for file in src[ProtoInfo].transitive_imports.to_list()`
	`236`	`+ for file in src[ProtoInfo].transitive_sources.to_list()`
`237`	`237`	`]`
`238`	`238`
`239`	`239`	`def get_proto_arguments(protos, genfiles_dir_path):`