Changes to generate NextGen protos in Ads library

.github/workflows/tests.yaml

@@ -53,10 +53,11 @@ jobs:
         run: nox -s mypy-${{ matrix.python }}
   showcase:
     strategy:
+      fail-fast: false
       # Run showcase tests on the lowest and highest supported runtimes
       matrix:
         # TODO(https://github.com/googleapis/gapic-generator-python/issues/2121) Remove `showcase_w_rest_async` target when async rest is GA.
-        python: ["3.7", "3.13"]
+        python: ["3.8", "3.13"]
         target: [showcase, showcase_alternative_templates, showcase_w_rest_async]
     runs-on: ubuntu-latest
     steps:
@@ -139,8 +140,9 @@ jobs:
   # TODO(yon-mg): add compute unit tests
   showcase-unit:
     strategy:
+      fail-fast: false
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         # TODO(https://github.com/googleapis/gapic-generator-python/issues/2121) Remove `_w_rest_async` variant when async rest is GA.
         variant: ['', _alternative_templates, _mixins, _alternative_templates_mixins, _w_rest_async]
     runs-on: ubuntu-latest
@@ -240,7 +242,7 @@ jobs:
   unit:
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -261,7 +263,7 @@ jobs:
   fragment:
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         variant: ['', _alternative_templates]
     runs-on: ubuntu-latest
     steps:

gapic/generator/generator.py

@@ -274,6 +274,8 @@ def _render_template(
                         and proto.meta.address.subpackage != api_schema.subpackage_view
                 ):
                     continue
+                if api_schema.all_library_settings[api_schema.naming.proto_package].python_settings.experimental_features.protobuf_pythonic_types_enabled:
+                    continue
 
                 answer.update(
                     self._get_file(

gapic/schema/metadata.py

@@ -115,6 +115,9 @@ def is_proto_plus_type(self) -> bool:
         Returns:
             bool: Whether the given package uses proto-plus types or not.
         """
+        if self.proto_package.startswith(self.api_naming.proto_package) and hasattr(self.api_naming, "protobuf_pythonic_types_enabled") and self.api_naming.protobuf_pythonic_types_enabled:
+            return False
+
         return self.proto_package.startswith(self.api_naming.proto_package) or (
             hasattr(self.api_naming, "proto_plus_deps")
             and self.proto_package in self.api_naming.proto_plus_deps
@@ -204,15 +207,16 @@ def python_import(self) -> imp.Import:
                 alias=self.module_alias,
             )
 
+        use_protobuf_types = hasattr(self.api_naming, "protobuf_pythonic_types_enabled") and self.api_naming.protobuf_pythonic_types_enabled
         # If this is part of the proto package that we are generating,
         # rewrite the package to our structure.
         if self.proto_package.startswith(self.api_naming.proto_package):
             return imp.Import(
                 package=self.api_naming.module_namespace + (
                     self.api_naming.versioned_module_name,
                 ) + self.subpackage + ('types',),
-                module=self.module,
-                alias=self.module_alias,
+                module= self.module + ('_pb2' if use_protobuf_types else ''),
+                alias='' if use_protobuf_types else self.module_alias,
             )
 
         if self.is_proto_plus_type:
@@ -240,10 +244,11 @@ def sphinx(self) -> str:
 
         # Check if this is a generated type
         # Use the original module name rather than the module_alias
+        use_protobuf_types = hasattr(self.api_naming, "protobuf_pythonic_types_enabled") and self.api_naming.protobuf_pythonic_types_enabled
         if self.proto_package.startswith(self.api_naming.proto_package):
             return '.'.join(self.api_naming.module_namespace + (
                 self.api_naming.versioned_module_name,
-            ) + self.subpackage + ('types',) + self.parent + (self.name, ))
+            ) + self.subpackage + ('types',) + self.parent + (self.name + ("_pb2" if use_protobuf_types else ""), ))
         elif self.is_proto_plus_type:
             return ".".join(
                 self.convert_to_versioned_package()

gapic/schema/naming.py

@@ -44,6 +44,7 @@ class Naming(abc.ABC):
     proto_package: str = ''
     _warehouse_package_name: str = ''
     proto_plus_deps: Tuple[str, ...] = dataclasses.field(default_factory=tuple)
+    protobuf_pythonic_types_enabled: bool = False
 
     def __post_init__(self):
         if not self.product_name:
@@ -152,7 +153,18 @@ def build(
                 package_info,
                 proto_plus_deps=opts.proto_plus_deps,
             )
-
+        package = ".".join(package_info.module_namespace) + f".{package_info.module_name}.{package_info.version}"
+
+        if "publishing" in opts.service_yaml_config:
+            if "library_settings" in opts.service_yaml_config["publishing"]:
+                for setting in opts.service_yaml_config["publishing"]["library_settings"]:
+                    if package == setting["version"]:
+                        if "protobuf_pythonic_types_enabled" in setting["python_settings"]["experimental_features"]:
+                            package_info = dataclasses.replace(
+                                package_info,
+                                protobuf_pythonic_types_enabled = setting["python_settings"]["experimental_features"]["protobuf_pythonic_types_enabled"]
+                            )
+
         # Done; return the naming information.
         return package_info
 

gapic/templates/%namespace/%name_%version/%sub/__init__.py.j2

@@ -2,6 +2,7 @@
 
 {% block content %}
 
+{% set protobuf_pythonic_types_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.protobuf_pythonic_types_enabled %}
 {% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}
 from {{package_path}} import gapic_version as package_version
 
@@ -29,10 +30,10 @@ from .services.{{ service.name|snake_case }} import {{ service.async_client_name
 {% for proto in api.protos.values()|sort(attribute='name')
         if proto.meta.address.subpackage == api.subpackage_view %}
 {% for message in proto.messages.values()|sort(attribute='name') %}
-from .types.{{ proto.module_name }} import {{ message.name }}
+from .types.{{ proto.module_name }}{%if protobuf_pythonic_types_enabled %}_pb2{% endif %} import {{ message.name }}
 {% endfor %}
 {% for enum in proto.enums.values()|sort(attribute='name') %}
-from .types.{{ proto.module_name }} import {{ enum.name }}
+from .types.{{ proto.module_name }}{%if protobuf_pythonic_types_enabled %}_pb2{% endif %} import {{ enum.name }}
 {% endfor %}
 {% endfor %}
 

gapic/templates/%namespace/%name_%version/%sub/services/%service/_client_macros.j2

@@ -17,6 +17,8 @@
 {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
 
 {% macro client_method(method, name, snippet_index, api, service, full_extended_lro=False) %}
+    {% set protobuf_pythonic_types_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.protobuf_pythonic_types_enabled %}
+
     def {{ name }}(self,
             {% if not method.client_streaming %}
             request: Optional[Union[{{ method.input.ident }}, dict]] = None,
@@ -97,7 +99,7 @@
                              'the individual field arguments should be set.')
 
         {% endif %}
-        {% if method.input.ident.package != method.ident.package %}{# request lives in a different package, so there is no proto wrapper #}
+        {% if protobuf_pythonic_types_enabled or method.input.ident.package != method.ident.package %}{# request lives in a different package, so there is no proto wrapper #}
         if isinstance(request, dict):
             # - The request isn't a proto-plus wrapped type,
             #   so it must be constructed via keyword expansion.
@@ -113,22 +115,26 @@
         {% endif %}{# different request package #}
 
             {#- Vanilla python protobuf wrapper types cannot _set_ repeated fields #}
-            {% if method.flattened_fields and method.input.ident.package == method.ident.package %}
+            {% if not protobuf_pythonic_types_enabled and method.flattened_fields and method.input.ident.package == method.ident.package %}
             # If we have keyword arguments corresponding to fields on the
             # request, apply these.
             {% endif %}
-            {% for key, field in method.flattened_fields.items() if not field.repeated or method.input.ident.package == method.ident.package %}
+            {% for key, field in method.flattened_fields.items() if not field.repeated or (not protobuf_pythonic_types_enabled and method.input.ident.package == method.ident.package) %}
             if {{ field.name }} is not None:
                 {# Repeated values is a special case, because values can be lists. #}
                 {# In order to not confuse the marshalling logic, extend these fields instead of assigning #}
                 {% if field.ident.ident|string() == "struct_pb2.Value" and field.repeated %}
                 request.{{ key }}.extend({{ field.name }})
                 {% else %}
+                {% if protobuf_pythonic_types_enabled and field.message %}
+                request.{{ key }}.MergeFrom({{ key }})
+                {% else %}
                 request.{{ key }} = {{ field.name }}
+                {% endif %}
                 {% endif %}{# struct_pb2.Value #}
             {% endfor %}
             {# Map-y fields can be _updated_, however #}
-            {% for key, field in method.flattened_fields.items() if field.repeated and method.input.ident.package != method.ident.package %}
+            {% for key, field in method.flattened_fields.items() if field.repeated and (protobuf_pythonic_types_enabled or method.input.ident.package != method.ident.package) %}
             {% if field.map %} {# map implies repeated, but repeated does NOT imply map#}
             if {{ field.name }}:
                 request.{{ key }}.update({{ field.name }})
@@ -138,7 +144,7 @@
                 request.{{ key }}.extend({{ field.name }})
             {% endif %} {# field.map #}
             {% endfor %} {# method.flattened_fields.items() #}
-            {% endif %} {# method.client_streaming #}
+            {% endif %} {# method.client_streaming #}    
 
         # Wrap the RPC method; this adds retry and timeout information,
         # and friendly error handling.

gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2

@@ -4,6 +4,8 @@
 {% import "%namespace/%name_%version/%sub/services/%service/_client_macros.j2" as macros %}
 {% import "%namespace/%name_%version/%sub/services/%service/_shared_macros.j2" as shared_macros %}
 
+{% set protobuf_pythonic_types_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.protobuf_pythonic_types_enabled %}
+
 from collections import OrderedDict
 import re
 from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, {% if service.any_server_streaming %}AsyncIterable, Awaitable, {% endif %}{% if service.any_client_streaming %}AsyncIterator, {% endif %}Sequence, Tuple, Type, Union
@@ -52,7 +54,6 @@ from .transports.base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO
 from .transports.grpc_asyncio import {{ service.grpc_asyncio_transport_name }}
 from .client import {{ service.client_name }}
 
-
 {# TODO(yon-mg): handle rest transport async client interaction #}
 class {{ service.async_client_name }}:
     """{{ service.meta.doc|rst(width=72, indent=4) }}{% if service.version|length %}
@@ -324,7 +325,7 @@ class {{ service.async_client_name }}:
                              "the individual field arguments should be set.")
 
         {% endif %}
-        {% if method.input.ident.package != method.ident.package %} {# request lives in a different package, so there is no proto wrapper #}
+        {% if protobuf_pythonic_types_enabled or method.input.ident.package != method.ident.package %} {# request lives in a different package, so there is no proto wrapper #}
         # - The request isn't a proto-plus wrapped type,
         #   so it must be constructed via keyword expansion.
         if isinstance(request, dict):
@@ -338,27 +339,37 @@ class {{ service.async_client_name }}:
             request = {{ method.input.ident }}(request)
         {% endif %} {# different request package #}
 
-        {# Vanilla python protobuf wrapper types cannot _set_ repeated fields #}
-        {% if method.flattened_fields and method.input.ident.package == method.ident.package %}
-        # If we have keyword arguments corresponding to fields on the
-        # request, apply these.
-        {% endif %}
-        {% for key, field in method.flattened_fields.items() if not field.repeated and method.input.ident.package == method.ident.package %}
-        if {{ field.name }} is not None:
-            request.{{ key }} = {{ field.name }}
-        {% endfor %}
-        {# Map-y fields can be _updated_, however #}
-        {% for key, field in method.flattened_fields.items() if field.map and method.input.ident.package == method.ident.package %}
-
-        if {{ field.name }}:
-            request.{{ key }}.update({{ field.name }})
-        {% endfor %}
-        {# And list-y fields can be _extended_ #}
-        {% for key, field in method.flattened_fields.items() if field.repeated and not field.map and method.input.ident.package == method.ident.package %}
-        if {{ field.name }}:
-            request.{{ key }}.extend({{ field.name }})
-        {% endfor %}
-        {% endif %}
+            {# Vanilla python protobuf wrapper types cannot _set_ repeated fields #}
+            {% if not protobuf_pythonic_types_enabled and method.flattened_fields and method.input.ident.package == method.ident.package %}
+            # If we have keyword arguments corresponding to fields on the
+            # request, apply these.
+            {% endif %}
+            {% for key, field in method.flattened_fields.items() if not field.repeated or (not protobuf_pythonic_types_enabled and method.input.ident.package == method.ident.package) %}
+            if {{ field.name }} is not None:
+                {# Repeated values is a special case, because values can be lists. #}
+                {# In order to not confuse the marshalling logic, extend these fields instead of assigning #}
+                {% if field.ident.ident|string() == "struct_pb2.Value" and field.repeated %}
+                request.{{ key }}.extend({{ field.name }})
+                {% else %}
+                {% if protobuf_pythonic_types_enabled and field.message %}
+                request.{{ key }}.MergeFrom({{ key }})
+                {% else %}
+                request.{{ key }} = {{ field.name }}
+                {% endif %}
+                {% endif %}{# struct_pb2.Value #}
+            {% endfor %}
+            {# Map-y fields can be _updated_, however #}
+            {% for key, field in method.flattened_fields.items() if field.repeated and (protobuf_pythonic_types_enabled or method.input.ident.package != method.ident.package) %}
+            {% if field.map %} {# map implies repeated, but repeated does NOT imply map#}
+            if {{ field.name }}:
+                request.{{ key }}.update({{ field.name }})
+            {% else %}
+            {# And list-y fields can be _extended_ #}
+            if {{ field.name }}:
+                request.{{ key }}.extend({{ field.name }})
+            {% endif %} {# field.map #}
+            {% endfor %} {# method.flattened_fields.items() #}
+            {% endif %} {# method.client_streaming #}    
 
         # Wrap the RPC method; this adds retry and timeout information,
         # and friendly error handling.