From d5d13404966a11f5df49e2f71280b83caea2fc28 Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Fri, 25 Sep 2020 12:25:54 -0700 Subject: [PATCH 1/2] feat: add support for common resource paths Google Cloud defines a small set of common resources that do not belong to specific APIs or message types. All generated service clients now contain helper methods that allow construction and parsing of these paths. See https://github.com/googleapis/googleapis/blob/master/google/cloud/common_resources.proto for the list of common resources for Google Cloud. --- .../%sub/services/%service/client.py.j2 | 13 +++++ .../%name_%version/%sub/test_%service.py.j2 | 27 ++++++++++- gapic/schema/wrappers.py | 47 ++++++++++++++++++- .../%sub/services/%service/async_client.py.j2 | 4 ++ .../%sub/services/%service/client.py.j2 | 15 +++++- .../%name_%version/%sub/test_%service.py.j2 | 27 ++++++++++- tests/unit/schema/wrappers/test_service.py | 41 ++++++++++++++++ 7 files changed, 167 insertions(+), 7 deletions(-) diff --git a/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2 b/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2 index 2acc770d91..db181fabb8 100644 --- a/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2 +++ b/gapic/ads-templates/%namespace/%name/%version/%sub/services/%service/client.py.j2 @@ -132,6 +132,19 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): m = re.match(r"{{ message.path_regex_str }}", path) return m.groupdict() if m else {} {% endfor %} + {% for resource_msg in service.common_resources|sort(attribute="type_name") -%} + @staticmethod + def common_{{ resource_msg.message_type.resource_type|snake_case }}_path({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}: str, {%endfor %}) -> str: + """Return a fully-qualified {{ resource_msg.message_type.resource_type|snake_case }} string.""" + return "{{ resource_msg.message_type.resource_path }}".format({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}={{ arg }}, {% endfor %}) + + @staticmethod + def parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(path: str) -> Dict[str,str]: + """Parse a {{ resource_msg.message_type.resource_type|snake_case }} path into its component segments.""" + m = re.match(r"{{ resource_msg.message_type.path_regex_str }}", path) + return m.groupdict() if m else {} + + {% endfor %} {# common resources #} def __init__(self, *, credentials: Optional[credentials.Credentials] = None, diff --git a/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index 43a8e02814..a342db6f1c 100644 --- a/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/gapic/ads-templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -714,8 +714,8 @@ def test_{{ service.name|snake_case }}_grpc_lro_client(): {% endif -%} -{% for message in service.resource_messages|sort(attribute="resource_type") -%} {% with molluscs = cycler("squid", "clam", "whelk", "octopus", "oyster", "nudibranch", "cuttlefish", "mussel", "winkle", "nautilus", "scallop", "abalone") -%} +{% for message in service.resource_messages|sort(attribute="resource_type") -%} def test_{{ message.resource_type|snake_case }}_path(): {% for arg in message.resource_path_args -%} {{ arg }} = "{{ molluscs.next() }}" @@ -737,8 +737,31 @@ def test_parse_{{ message.resource_type|snake_case }}_path(): actual = {{ service.client_name }}.parse_{{ message.resource_type|snake_case }}_path(path) assert expected == actual -{% endwith -%} {% endfor -%} +{% for resource_msg in service.common_resources -%} +def test_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(): + {% for arg in resource_msg.message_type.resource_path_args -%} + {{ arg }} = "{{ molluscs.next() }}" + {% endfor %} + expected = "{{ resource_msg.message_type.resource_path }}".format({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}={{ arg }}, {% endfor %}) + actual = {{ service.client_name }}.common_{{ resource_msg.message_type.resource_type|snake_case }}_path({{ resource_msg.message_type.resource_path_args|join(", ") }}) + assert expected == actual + + +def test_parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(): + expected = { + {% for arg in resource_msg.message_type.resource_path_args -%} + "{{ arg }}": "{{ molluscs.next() }}", + {% endfor %} + } + path = {{ service.client_name }}.common_{{ resource_msg.message_type.resource_type|snake_case }}_path(**expected) + + # Check that the path construction is reversible. + actual = {{ service.client_name }}.parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(path) + assert expected == actual + +{% endfor -%} {# common resources#} +{% endwith -%} {# cycler #} def test_client_withDEFAULT_CLIENT_INFO(): client_info = gapic_v1.client_info.ClientInfo() diff --git a/gapic/schema/wrappers.py b/gapic/schema/wrappers.py index 7447607170..9ada03f75b 100644 --- a/gapic/schema/wrappers.py +++ b/gapic/schema/wrappers.py @@ -31,8 +31,8 @@ import dataclasses import re from itertools import chain -from typing import (cast, Dict, FrozenSet, Iterable, List, Mapping, Optional, - Sequence, Set, Tuple, Union) +from typing import (cast, Dict, FrozenSet, Iterable, List, Mapping, + ClassVar, Optional, Sequence, Set, Tuple, Union) from google.api import annotations_pb2 # type: ignore from google.api import client_pb2 @@ -855,6 +855,26 @@ def with_context(self, *, collisions: FrozenSet[str]) -> 'Method': ) +@dataclasses.dataclass(frozen=True) +class CommonResource: + type_name: str + pattern: str + + @utils.cached_property + def message_type(self): + message_pb = descriptor_pb2.DescriptorProto() + res_pb = message_pb.options.Extensions[resource_pb2.resource] + res_pb.type = self.type_name + res_pb.pattern.append(self.pattern) + + return MessageType( + message_pb=message_pb, + fields={}, + nested_enums={}, + nested_messages={}, + ) + + @dataclasses.dataclass(frozen=True) class Service: """Description of a service (defined with the ``service`` keyword).""" @@ -864,6 +884,29 @@ class Service: default_factory=metadata.Metadata, ) + common_resources: ClassVar[Sequence[CommonResource]] = ( + CommonResource( + "cloudresourcemanager.googleapis.com/Project", + "projects/{project}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Organization", + "organizations/{organization}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Folder", + "folders/{folder}", + ), + CommonResource( + "cloudbilling.googleapis.com/BillingAccount", + "billingAccounts/{billing_account}", + ), + CommonResource( + "locations.googleapis.com/Location", + "projects/{project}/locations/{location}", + ), + ) + def __getattr__(self, name): return getattr(self.service_pb, name) diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 index 95a250479f..772643b448 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/async_client.py.j2 @@ -42,6 +42,10 @@ class {{ service.async_client_name }}: {{ message.resource_type|snake_case }}_path = staticmethod({{ service.client_name }}.{{ message.resource_type|snake_case }}_path) parse_{{ message.resource_type|snake_case}}_path = staticmethod({{ service.client_name }}.parse_{{ message.resource_type|snake_case }}_path) {% endfor %} + {% for resource_msg in service.common_resources %} + common_{{ resource_msg.message_type.resource_type|snake_case }}_path = staticmethod({{ service.client_name }}.common_{{ resource_msg.message_type.resource_type|snake_case }}_path) + parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path = staticmethod({{ service.client_name }}.parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path) + {% endfor %} from_service_account_file = {{ service.client_name }}.from_service_account_file from_service_account_json = from_service_account_file diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 index ccc4aa85f0..d709792514 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 @@ -137,7 +137,20 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta): """Parse a {{ message.resource_type|snake_case }} path into its component segments.""" m = re.match(r"{{ message.path_regex_str }}", path) return m.groupdict() if m else {} - {% endfor %} + {% endfor %} {# resources #} + {% for resource_msg in service.common_resources|sort(attribute="type_name") -%} + @staticmethod + def common_{{ resource_msg.message_type.resource_type|snake_case }}_path({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}: str, {%endfor %}) -> str: + """Return a fully-qualified {{ resource_msg.message_type.resource_type|snake_case }} string.""" + return "{{ resource_msg.message_type.resource_path }}".format({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}={{ arg }}, {% endfor %}) + + @staticmethod + def parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(path: str) -> Dict[str,str]: + """Parse a {{ resource_msg.message_type.resource_type|snake_case }} path into its component segments.""" + m = re.match(r"{{ resource_msg.message_type.path_regex_str }}", path) + return m.groupdict() if m else {} + + {% endfor %} {# common resources #} def __init__(self, *, credentials: Optional[credentials.Credentials] = None, diff --git a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index 4d5b934151..c74d49c102 100644 --- a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -1304,8 +1304,8 @@ def test_{{ service.name|snake_case }}_grpc_lro_async_client(): {% endif -%} -{% for message in service.resource_messages|sort(attribute="resource_type") -%} {% with molluscs = cycler("squid", "clam", "whelk", "octopus", "oyster", "nudibranch", "cuttlefish", "mussel", "winkle", "nautilus", "scallop", "abalone") -%} +{% for message in service.resource_messages|sort(attribute="resource_type") -%} def test_{{ message.resource_type|snake_case }}_path(): {% for arg in message.resource_path_args -%} {{ arg }} = "{{ molluscs.next() }}" @@ -1327,8 +1327,31 @@ def test_parse_{{ message.resource_type|snake_case }}_path(): actual = {{ service.client_name }}.parse_{{ message.resource_type|snake_case }}_path(path) assert expected == actual -{% endwith -%} {% endfor -%} +{% for resource_msg in service.common_resources -%} +def test_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(): + {% for arg in resource_msg.message_type.resource_path_args -%} + {{ arg }} = "{{ molluscs.next() }}" + {% endfor %} + expected = "{{ resource_msg.message_type.resource_path }}".format({% for arg in resource_msg.message_type.resource_path_args %}{{ arg }}={{ arg }}, {% endfor %}) + actual = {{ service.client_name }}.common_{{ resource_msg.message_type.resource_type|snake_case }}_path({{ resource_msg.message_type.resource_path_args|join(", ") }}) + assert expected == actual + + +def test_parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(): + expected = { + {% for arg in resource_msg.message_type.resource_path_args -%} + "{{ arg }}": "{{ molluscs.next() }}", + {% endfor %} + } + path = {{ service.client_name }}.common_{{ resource_msg.message_type.resource_type|snake_case }}_path(**expected) + + # Check that the path construction is reversible. + actual = {{ service.client_name }}.parse_common_{{ resource_msg.message_type.resource_type|snake_case }}_path(path) + assert expected == actual + +{% endfor -%} {# common resources#} +{% endwith -%} {# cycler #} def test_client_withDEFAULT_CLIENT_INFO(): diff --git a/tests/unit/schema/wrappers/test_service.py b/tests/unit/schema/wrappers/test_service.py index 8502617b5d..d733d1fa2d 100644 --- a/tests/unit/schema/wrappers/test_service.py +++ b/tests/unit/schema/wrappers/test_service.py @@ -20,6 +20,7 @@ from google.protobuf import descriptor_pb2 from gapic.schema import imp +from gapic.schema.wrappers import CommonResource from test_utils.test_utils import ( get_method, @@ -295,3 +296,43 @@ def test_has_pagers(): ), ) assert not other_service.has_pagers + + +def test_default_common_resources(): + service = make_service(name="MolluscMaker") + + assert service.common_resources == ( + CommonResource( + "cloudresourcemanager.googleapis.com/Project", + "projects/{project}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Organization", + "organizations/{organization}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Folder", + "folders/{folder}", + ), + CommonResource( + "cloudbilling.googleapis.com/BillingAccount", + "billingAccounts/{billing_account}", + ), + CommonResource( + "locations.googleapis.com/Location", + "projects/{project}/locations/{location}", + ), + ) + + +def test_common_resource_patterns(): + species = CommonResource( + "nomenclature.linnaen.com/Species", + "families/{family}/genera/{genus}/species/{species}", + ) + species_msg = species.message_type + + assert species_msg.resource_path == "families/{family}/genera/{genus}/species/{species}" + assert species_msg.resource_type == "Species" + assert species_msg.resource_path_args == ["family", "genus", "species"] + assert species_msg.path_regex_str == '^families/(?P.+?)/genera/(?P.+?)/species/(?P.+?)$' From dab540f286e1074581a2e3e896156653226e4863 Mon Sep 17 00:00:00 2001 From: Dov Shlachter Date: Mon, 28 Sep 2020 09:10:38 -0700 Subject: [PATCH 2/2] Satiate 3.6 --- gapic/schema/wrappers.py | 44 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/gapic/schema/wrappers.py b/gapic/schema/wrappers.py index 9ada03f75b..68ec4e2316 100644 --- a/gapic/schema/wrappers.py +++ b/gapic/schema/wrappers.py @@ -884,27 +884,31 @@ class Service: default_factory=metadata.Metadata, ) - common_resources: ClassVar[Sequence[CommonResource]] = ( - CommonResource( - "cloudresourcemanager.googleapis.com/Project", - "projects/{project}", - ), - CommonResource( - "cloudresourcemanager.googleapis.com/Organization", - "organizations/{organization}", - ), - CommonResource( - "cloudresourcemanager.googleapis.com/Folder", - "folders/{folder}", - ), - CommonResource( - "cloudbilling.googleapis.com/BillingAccount", - "billingAccounts/{billing_account}", - ), - CommonResource( - "locations.googleapis.com/Location", - "projects/{project}/locations/{location}", + common_resources: ClassVar[Sequence[CommonResource]] = dataclasses.field( + default=( + CommonResource( + "cloudresourcemanager.googleapis.com/Project", + "projects/{project}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Organization", + "organizations/{organization}", + ), + CommonResource( + "cloudresourcemanager.googleapis.com/Folder", + "folders/{folder}", + ), + CommonResource( + "cloudbilling.googleapis.com/BillingAccount", + "billingAccounts/{billing_account}", + ), + CommonResource( + "locations.googleapis.com/Location", + "projects/{project}/locations/{location}", + ), ), + init=False, + compare=False, ) def __getattr__(self, name):