feat: add partner_model_tuning_spec to TuningJob (#5721)

* feat: add a dynamic retrieval API

PiperOrigin-RevId: 680475495

Source-Link: googleapis/googleapis@a6a6956

Source-Link: googleapis/googleapis-gen@c9ef9e0
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6ImM5ZWY5ZTAwZjkwYWUyYzU3NGY4MDBiNjY2ZmU3NDg2MjIwYjliZjIifQ==

* feat: add psc_automation_configs to DeployIndex v1

PiperOrigin-RevId: 680742448

Source-Link: googleapis/googleapis@ce31830

Source-Link: googleapis/googleapis-gen@5254135
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6IjUyNTQxMzVjZDkwODlmMzVjOWY1MzdhYmE1NjdkZDI5YWNjZTU0MTMifQ==

* feat: add new `PscInterfaceConfig` field to `pipeline_job.proto`

PiperOrigin-RevId: 680823763

Source-Link: googleapis/googleapis@8f00dab

Source-Link: googleapis/googleapis-gen@e58485b
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6ImU1ODQ4NWI2NzA1NTBiNmI3ODM1MWVkZTQ5NmFkM2NlZDE1MDk1MzQifQ==

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: add enable_secure_private_service_connect in service attachment

PiperOrigin-RevId: 681172589

Source-Link: googleapis/googleapis@16a1580

Source-Link: googleapis/googleapis-gen@87fb5dc
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6Ijg3ZmI1ZGM3MzhhZGZhNWQzMTNhNTgzNjgxZTQ3NmUzNDcwZDY5ZTUifQ==

* feat: add continuous sync option in feature_view.proto

PiperOrigin-RevId: 681665991

Source-Link: googleapis/googleapis@8d17595

Source-Link: googleapis/googleapis-gen@ceb528d
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6ImNlYjUyOGQzNzk1MDAxM2UxZTEyZWRkNGIwZTk5YjJkMjIwN2M3ZjkifQ==

* feat: add a dynamic retrieval API

PiperOrigin-RevId: 681780557

Source-Link: googleapis/googleapis@475669f

Source-Link: googleapis/googleapis-gen@bbbb240
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6ImJiYmIyNDBhMzhiNGE0NDU3NDUxZjYzYTNlZmFjYmI4MDY4YTY3ZWQifQ==

* feat: add partner_model_tuning_spec to TuningJob

PiperOrigin-RevId: 683655446

Source-Link: googleapis/googleapis@4be7103

Source-Link: googleapis/googleapis-gen@2f7b2ea
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWFpcGxhdGZvcm0vLk93bEJvdC55YW1sIiwiaCI6IjJmN2IyZWE2NDMyMGNlYTFjYWVhYzMwMTYzNWY1MzEzODUwYzk5ZDgifQ==

* chore: run post-processor

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: sofisl <55454395+sofisl@users.noreply.github.com>
Co-authored-by: Sofia Leon <sofialeon@google.com>

Author: gcf-owl-bot[bot]

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/content.proto

@@ -560,6 +560,12 @@ message GroundingMetadata {
   // Optional. List of grounding support.
   repeated GroundingSupport grounding_supports = 6
       [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Output only. Retrieval metadata.
+  optional RetrievalMetadata retrieval_metadata = 7 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.field_behavior) = OUTPUT_ONLY
+  ];
 }
 
 // Google search entry point.
@@ -572,3 +578,14 @@ message SearchEntryPoint {
   // url> tuple.
   bytes sdk_blob = 2 [(google.api.field_behavior) = OPTIONAL];
 }
+
+// Metadata related to retrieval in the grounding flow.
+message RetrievalMetadata {
+  // Optional. Score indicating how likely information from google search could
+  // help answer the prompt. The score is in the range `[0, 1]`, where 0 is the
+  // least likely and 1 is the most likely. This score is only populated when
+  // google search grounding and dynamic retrieval is enabled. It will be
+  // compared to the threshold to determine whether to trigger google search.
+  float google_search_dynamic_retrieval_score = 2
+      [(google.api.field_behavior) = OPTIONAL];
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/feature_view.proto

@@ -55,6 +55,10 @@ message FeatureView {
     // database. For example, "CRON_TZ=America/New_York 1 * * * *", or
     // "TZ=America/New_York 1 * * * *".
     string cron = 1;
+
+    // Optional. If true, syncs the FeatureView in a continuous manner to Online
+    // Store.
+    bool continuous = 2 [(google.api.field_behavior) = OPTIONAL];
   }
 
   // Configuration for vector indexing.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/index_endpoint.proto

@@ -260,6 +260,12 @@ message DeployedIndex {
   //
   // Note: we only support up to 5 deployment groups(not including 'default').
   string deployment_group = 11 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. If set for PSC deployed index, PSC connection will be
+  // automatically created after deployment is done and the endpoint information
+  // is populated in private_endpoints.psc_automated_endpoints.
+  repeated PSCAutomationConfig psc_automation_configs = 19
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Used to set up the auth on the DeployedIndex's private endpoint.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/service_networking.proto

@@ -31,6 +31,21 @@ option (google.api.resource_definition) = {
   pattern: "projects/{project}/regions/{region}/networkAttachments/{networkattachment}"
 };
 
+// PSC config that is used to automatically create forwarding rule via
+// ServiceConnectionMap.
+message PSCAutomationConfig {
+  // Required. Project id used to create forwarding rule.
+  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. The full name of the Google Compute Engine
+  // [network](https://cloud.google.com/compute/docs/networks-and-firewalls#networks).
+  // [Format](https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert):
+  // `projects/{project}/global/networks/{network}`.
+  // Where {project} is a project number, as in '12345', and {network} is
+  // network name.
+  string network = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
 // Represents configuration for private service connect.
 message PrivateServiceConnectConfig {
   // Required. If true, expose the IndexEndpoint via private service connect.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/tool.proto

@@ -142,7 +142,29 @@ message VertexAISearch {
 }
 
 // Tool to retrieve public web data for grounding, powered by Google.
-message GoogleSearchRetrieval {}
+message GoogleSearchRetrieval {
+  // Specifies the dynamic retrieval configuration for the given source.
+  DynamicRetrievalConfig dynamic_retrieval_config = 2;
+}
+
+// Describes the options to customize dynamic retrieval.
+message DynamicRetrievalConfig {
+  // The mode of the predictor to be used in dynamic retrieval.
+  enum Mode {
+    // Always trigger retrieval.
+    MODE_UNSPECIFIED = 0;
+
+    // Run retrieval only when system decides it is necessary.
+    MODE_DYNAMIC = 1;
+  }
+
+  // The mode of the predictor to be used in dynamic retrieval.
+  Mode mode = 1;
+
+  // Optional. The threshold to be used in dynamic retrieval.
+  // If not set, a system default value is used.
+  optional float dynamic_threshold = 2 [(google.api.field_behavior) = OPTIONAL];
+}
 
 // Tool config. This config is shared for all tools provided in the request.
 message ToolConfig {

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/content.proto

@@ -561,6 +561,12 @@ message GroundingMetadata {
   // Optional. List of grounding support.
   repeated GroundingSupport grounding_supports = 6
       [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Output only. Retrieval metadata.
+  optional RetrievalMetadata retrieval_metadata = 7 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.field_behavior) = OUTPUT_ONLY
+  ];
 }
 
 // Google search entry point.
@@ -573,3 +579,14 @@ message SearchEntryPoint {
   // url> tuple.
   bytes sdk_blob = 2 [(google.api.field_behavior) = OPTIONAL];
 }
+
+// Metadata related to retrieval in the grounding flow.
+message RetrievalMetadata {
+  // Optional. Score indicating how likely information from google search could
+  // help answer the prompt. The score is in the range [0, 1], where 0 is the
+  // least likely and 1 is the most likely. This score is only populated when
+  // google search grounding and dynamic retrieval is enabled. It will be
+  // compared to the threshold to determine whether to trigger google search.
+  float google_search_dynamic_retrieval_score = 2
+      [(google.api.field_behavior) = OPTIONAL];
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/pipeline_job.proto

@@ -24,6 +24,7 @@ import "google/cloud/aiplatform/v1beta1/encryption_spec.proto";
 import "google/cloud/aiplatform/v1beta1/execution.proto";
 import "google/cloud/aiplatform/v1beta1/pipeline_failure_policy.proto";
 import "google/cloud/aiplatform/v1beta1/pipeline_state.proto";
+import "google/cloud/aiplatform/v1beta1/service_networking.proto";
 import "google/cloud/aiplatform/v1beta1/ui_pipeline_spec.proto";
 import "google/cloud/aiplatform/v1beta1/value.proto";
 import "google/protobuf/struct.proto";
@@ -198,6 +199,10 @@ message PipelineJob {
   // Example: ['vertex-ai-ip-range'].
   repeated string reserved_ip_ranges = 25;
 
+  // Optional. Configuration for PSC-I for PipelineJob.
+  PscInterfaceConfig psc_interface_config = 31
+      [(google.api.field_behavior) = OPTIONAL];
+
   // A template uri from where the
   // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1beta1.PipelineJob.pipeline_spec],
   // if empty, will be downloaded. Currently, only uri from Vertex Template

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/service_networking.proto

@@ -56,6 +56,13 @@ message PrivateServiceConnectConfig {
   // attachment.
   repeated string project_allowlist = 2;
 
+  // Optional. If set to true, enable secure private service connect with IAM
+  // authorization. Otherwise, private service connect will be done without
+  // authorization. Note latency will be slightly increased if authorization is
+  // enabled.
+  bool enable_secure_private_service_connect = 4
+      [(google.api.field_behavior) = OPTIONAL];
+
   // Output only. The name of the generated service attachment resource.
   // This is only populated if the endpoint is deployed with
   // PrivateServiceConnect.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/tool.proto

@@ -233,7 +233,29 @@ message VertexAISearch {
 }
 
 // Tool to retrieve public web data for grounding, powered by Google.
-message GoogleSearchRetrieval {}
+message GoogleSearchRetrieval {
+  // Specifies the dynamic retrieval configuration for the given source.
+  DynamicRetrievalConfig dynamic_retrieval_config = 2;
+}
+
+// Describes the options to customize dynamic retrieval.
+message DynamicRetrievalConfig {
+  // The mode of the predictor to be used in dynamic retrieval.
+  enum Mode {
+    // Always trigger retrieval.
+    MODE_UNSPECIFIED = 0;
+
+    // Run retrieval only when system decides it is necessary.
+    MODE_DYNAMIC = 1;
+  }
+
+  // The mode of the predictor to be used in dynamic retrieval.
+  Mode mode = 1;
+
+  // Optional. The threshold to be used in dynamic retrieval.
+  // If not set, a system default value is used.
+  optional float dynamic_threshold = 2 [(google.api.field_behavior) = OPTIONAL];
+}
 
 // Tool config. This config is shared for all tools provided in the request.
 message ToolConfig {

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/tuning_job.proto

@@ -21,6 +21,7 @@ import "google/api/resource.proto";
 import "google/cloud/aiplatform/v1beta1/content.proto";
 import "google/cloud/aiplatform/v1beta1/encryption_spec.proto";
 import "google/cloud/aiplatform/v1beta1/job_state.proto";
+import "google/protobuf/struct.proto";
 import "google/protobuf/timestamp.proto";
 import "google/rpc/status.proto";
 
@@ -52,6 +53,9 @@ message TuningJob {
 
     // Tuning Spec for Distillation.
     DistillationSpec distillation_spec = 17;
+
+    // Tuning Spec for open sourced and third party Partner models.
+    PartnerModelTuningSpec partner_model_tuning_spec = 21;
   }
 
   // Output only. Identifier. Resource name of a TuningJob. Format:
@@ -451,6 +455,21 @@ message DistillationHyperParameters {
       [(google.api.field_behavior) = OPTIONAL];
 }
 
+// Tuning spec for Partner models.
+message PartnerModelTuningSpec {
+  // Required. Cloud Storage path to file containing training dataset for
+  // tuning. The dataset must be formatted as a JSONL file.
+  string training_dataset_uri = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Optional. Cloud Storage path to file containing validation dataset for
+  // tuning. The dataset must be formatted as a JSONL file.
+  string validation_dataset_uri = 2 [(google.api.field_behavior) = OPTIONAL];
+
+  // Hyperparameters for tuning. The accepted hyper_parameters and their valid
+  // range of values will differ depending on the base model.
+  map<string, google.protobuf.Value> hyper_parameters = 3;
+}
+
 // TunedModel Reference for legacy model migration.
 message TunedModelRef {
   // The Tuned Model Reference for the model.