Synchronize new proto/yaml changes.

PiperOrigin-RevId: 222456573
pull/530/head
Google APIs 6 years ago committed by Copybara-Service
parent 0aa59b1d28
commit 3cef6c237b
  1. 2
      google/cloud/dataproc/v1/BUILD.bazel
  2. 204
      google/cloud/dataproc/v1/clusters.proto
  3. 2
      google/cloud/dataproc/v1/dataproc.yaml
  4. 257
      google/cloud/dataproc/v1/dataproc_gapic.yaml
  5. 75
      google/cloud/dataproc/v1/jobs.proto
  6. 29
      google/cloud/dataproc/v1/operations.proto
  7. 662
      google/cloud/dataproc/v1/workflow_templates.proto

@ -19,6 +19,7 @@ proto_library(
"clusters.proto",
"jobs.proto",
"operations.proto",
"workflow_templates.proto",
],
deps = [
"//google/api:annotations_proto",
@ -80,6 +81,7 @@ java_gapic_library(
) for test_name in [
"com.google.cloud.dataproc.v1.ClusterControllerClientTest",
"com.google.cloud.dataproc.v1.JobControllerClientTest",
"com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientTest",
]]
##############################################################################

@ -1,4 +1,4 @@
// Copyright 2017 Google Inc.
// Copyright 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -11,13 +11,13 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/cloud/dataproc/v1/operations.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/field_mask.proto";
@ -30,43 +30,58 @@ option java_package = "com.google.cloud.dataproc.v1";
// The ClusterControllerService provides methods to manage clusters
// of Google Compute Engine instances.
// of Compute Engine instances.
service ClusterController {
// Creates a cluster in a project.
rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters" body: "cluster" };
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters"
body: "cluster"
};
}
// Updates a cluster in a project.
rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = { patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" body: "cluster" };
option (google.api.http) = {
patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
body: "cluster"
};
}
// Deletes a cluster in a project.
rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = { delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" };
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
};
}
// Gets the resource representation for a cluster in a project.
rpc GetCluster(GetClusterRequest) returns (Cluster) {
option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" };
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
};
}
// Lists all regions/{region}/clusters in a project.
rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) {
option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/clusters" };
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/clusters"
};
}
// Gets cluster diagnostic information.
// After the operation completes, the Operation.response field
// contains `DiagnoseClusterOutputLocation`.
rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose" body: "*" };
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
body: "*"
};
}
}
// Describes the identifying information, config, and status of
// a cluster of Google Compute Engine instances.
// a cluster of Compute Engine instances.
message Cluster {
// Required. The Google Cloud Platform project ID that the cluster belongs to.
string project_id = 1;
@ -87,13 +102,13 @@ message Cluster {
// No more than 32 labels can be associated with a cluster.
map<string, string> labels = 8;
// Output-only. Cluster status.
// Output only. Cluster status.
ClusterStatus status = 4;
// Output-only. The previous cluster status.
// Output only. The previous cluster status.
repeated ClusterStatus status_history = 7;
// Output-only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
// Output only. A cluster UUID (Unique Universal Identifier). Cloud Dataproc
// generates this value when it creates the cluster.
string cluster_uuid = 6;
@ -106,7 +121,7 @@ message Cluster {
// The cluster config.
message ClusterConfig {
// Optional. A Google Cloud Storage staging bucket used for sharing generated
// Optional. A Cloud Storage staging bucket used for sharing generated
// SSH keys and config. If you do not specify a staging bucket, Cloud
// Dataproc will determine an appropriate Cloud Storage location (US,
// ASIA, or EU) for your cluster's staging bucket according to the Google
@ -114,19 +129,19 @@ message ClusterConfig {
// and manage this project-level, per-location bucket for you.
string config_bucket = 1;
// Required. The shared Google Compute Engine config settings for
// Required. The shared Compute Engine config settings for
// all instances in a cluster.
GceClusterConfig gce_cluster_config = 8;
// Optional. The Google Compute Engine config settings for
// Optional. The Compute Engine config settings for
// the master instance in a cluster.
InstanceGroupConfig master_config = 9;
// Optional. The Google Compute Engine config settings for
// Optional. The Compute Engine config settings for
// worker instances in a cluster.
InstanceGroupConfig worker_config = 10;
// Optional. The Google Compute Engine config settings for
// Optional. The Compute Engine config settings for
// additional worker instances in a cluster.
InstanceGroupConfig secondary_worker_config = 12;
@ -145,12 +160,22 @@ message ClusterConfig {
// ... worker specific actions ...
// fi
repeated NodeInitializationAction initialization_actions = 11;
// Optional. Encryption settings for the cluster.
EncryptionConfig encryption_config = 15;
}
// Common config settings for resources of Google Compute Engine cluster
// Encryption settings for the cluster.
message EncryptionConfig {
// Optional. The Cloud KMS key name to use for PD disk encryption for all
// instances in the cluster.
string gce_pd_kms_key_name = 1;
}
// Common config settings for resources of Compute Engine cluster
// instances, applicable to all instances in the cluster.
message GceClusterConfig {
// Optional. The zone where the Google Compute Engine cluster will be located.
// Optional. The zone where the Compute Engine cluster will be located.
// On a create request, it is required in the "global" region. If omitted
// in a non-global Cloud Dataproc region, the service will pick a zone in the
// corresponding Compute Engine region. On a get request, zone will
@ -163,7 +188,7 @@ message GceClusterConfig {
// * `us-central1-f`
string zone_uri = 1;
// Optional. The Google Compute Engine network to be used for machine
// Optional. The Compute Engine network to be used for machine
// communications. Cannot be specified with subnetwork_uri. If neither
// `network_uri` nor `subnetwork_uri` is specified, the "default" network of
// the project is used, if it exists. Cannot be a "Custom Subnet Network" (see
@ -176,7 +201,7 @@ message GceClusterConfig {
// * `default`
string network_uri = 2;
// Optional. The Google Compute Engine subnetwork to be used for machine
// Optional. The Compute Engine subnetwork to be used for machine
// communications. Cannot be specified with network_uri.
//
// A full URL, partial URI, or short name are valid. Examples:
@ -195,8 +220,8 @@ message GceClusterConfig {
bool internal_ip_only = 7;
// Optional. The service account of the instances. Defaults to the default
// Google Compute Engine service account. Custom service accounts need
// permissions equivalent to the folloing IAM roles:
// Compute Engine service account. Custom service accounts need
// permissions equivalent to the following IAM roles:
//
// * roles/logging.logWriter
// * roles/storage.objectAdmin
@ -206,7 +231,7 @@ message GceClusterConfig {
// Example: `[account_id]@[project_id].iam.gserviceaccount.com`
string service_account = 8;
// Optional. The URIs of service account scopes to be included in Google
// Optional. The URIs of service account scopes to be included in
// Compute Engine instances. The following base set of scopes is always
// included:
//
@ -222,38 +247,43 @@ message GceClusterConfig {
// * https://www.googleapis.com/auth/devstorage.full_control
repeated string service_account_scopes = 3;
// The Google Compute Engine tags to add to all instances (see
// The Compute Engine tags to add to all instances (see
// [Tagging instances](/compute/docs/label-or-tag-resources#tags)).
repeated string tags = 4;
// The Google Compute Engine metadata entries to add to all instances (see
// The Compute Engine metadata entries to add to all instances (see
// [Project and instance metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
map<string, string> metadata = 5;
}
// Optional. The config settings for Google Compute Engine resources in
// Optional. The config settings for Compute Engine resources in
// an instance group, such as a master or worker group.
message InstanceGroupConfig {
// Optional. The number of VM instances in the instance group.
// For master instance groups, must be set to 1.
int32 num_instances = 1;
// Optional. The list of instance names. Cloud Dataproc derives the names from
// `cluster_name`, `num_instances`, and the instance group if not set by user
// (recommended practice is to let Cloud Dataproc derive the name).
// Output only. The list of instance names. Cloud Dataproc derives the names
// from `cluster_name`, `num_instances`, and the instance group.
repeated string instance_names = 2;
// Output-only. The Google Compute Engine image resource used for cluster
// instances. Inferred from `SoftwareConfig.image_version`.
// Optional. The Compute Engine image resource used for cluster
// instances. It can be specified or may be inferred from
// `SoftwareConfig.image_version`.
string image_uri = 3;
// Optional. The Google Compute Engine machine type used for cluster instances.
// Optional. The Compute Engine machine type used for cluster instances.
//
// A full URL, partial URI, or short name are valid. Examples:
//
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
// * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2`
// * `n1-standard-2`
//
// **Auto Zone Exception**: If you are using the Cloud Dataproc
// [Auto Zone Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
// feature, you must use the short name of the machine type
// resource, for example, `n1-standard-2`.
string machine_type_uri = 4;
// Optional. Disk option config settings.
@ -262,12 +292,12 @@ message InstanceGroupConfig {
// Optional. Specifies that this instance group contains preemptible instances.
bool is_preemptible = 6;
// Output-only. The config for Google Compute Engine Instance Group
// Output only. The config for Compute Engine Instance Group
// Manager that manages this group.
// This is only used for preemptible instance groups.
ManagedGroupConfig managed_group_config = 7;
// Optional. The Google Compute Engine accelerator configuration for these
// Optional. The Compute Engine accelerator configuration for these
// instances.
//
// **Beta Feature**: This feature is still under development. It may be
@ -277,25 +307,31 @@ message InstanceGroupConfig {
// Specifies the resources used to actively manage an instance group.
message ManagedGroupConfig {
// Output-only. The name of the Instance Template used for the Managed
// Output only. The name of the Instance Template used for the Managed
// Instance Group.
string instance_template_name = 1;
// Output-only. The name of the Instance Group Manager for this group.
// Output only. The name of the Instance Group Manager for this group.
string instance_group_manager_name = 2;
}
// Specifies the type and number of accelerator cards attached to the instances
// of an instance group (see [GPUs on Compute Engine](/compute/docs/gpus/)).
// of an instance. See [GPUs on Compute Engine](/compute/docs/gpus/).
message AcceleratorConfig {
// Full URL, partial URI, or short name of the accelerator type resource to
// expose to this instance. See [Google Compute Engine AcceleratorTypes](
// /compute/docs/reference/beta/acceleratorTypes)
// expose to this instance. See
// [Compute Engine AcceleratorTypes](/compute/docs/reference/beta/acceleratorTypes).
//
// Examples:
//
// Examples
// * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
// * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80`
// * `nvidia-tesla-k80`
//
// **Auto Zone Exception**: If you are using the Cloud Dataproc
// [Auto Zone Placement](/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
// feature, you must use the short name of the accelerator type
// resource, for example, `nvidia-tesla-k80`.
string accelerator_type_uri = 1;
// The number of the accelerator cards of this type exposed to this instance.
@ -304,6 +340,11 @@ message AcceleratorConfig {
// Specifies the config of disk options for a group of VM instances.
message DiskConfig {
// Optional. Type of the boot disk (default is "pd-standard").
// Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or
// "pd-standard" (Persistent Disk Hard Disk Drive).
string boot_disk_type = 3;
// Optional. Size in GB of the boot disk (default is 500GB).
int32 boot_disk_size_gb = 1;
@ -319,7 +360,7 @@ message DiskConfig {
// Specifies an executable to run on a fully configured node and a
// timeout period for executable completion.
message NodeInitializationAction {
// Required. Google Cloud Storage URI of executable file.
// Required. Cloud Storage URI of executable file.
string executable_file = 1;
// Optional. Amount of time executable has to complete. Default is
@ -352,7 +393,9 @@ message ClusterStatus {
UPDATING = 5;
}
// The cluster substate.
enum Substate {
// The cluster substate is unknown.
UNSPECIFIED = 0;
// The cluster is known to be in an unhealthy state
@ -369,25 +412,27 @@ message ClusterStatus {
STALE_STATUS = 2;
}
// Output-only. The cluster's state.
// Output only. The cluster's state.
State state = 1;
// Output-only. Optional details of cluster's state.
// Output only. Optional details of cluster's state.
string detail = 2;
// Output-only. Time when this state was entered.
// Output only. Time when this state was entered.
google.protobuf.Timestamp state_start_time = 3;
// Output-only. Additional state information that includes
// Output only. Additional state information that includes
// status reported by the agent.
Substate substate = 4;
}
// Specifies the selection and config of software inside the cluster.
message SoftwareConfig {
// Optional. The version of software inside the cluster. It must match the
// regular expression `[0-9]+\.[0-9]+`. If unspecified, it defaults to the
// latest version (see [Cloud Dataproc Versioning](/dataproc/versioning)).
// Optional. The version of software inside the cluster. It must be one of the supported
// [Cloud Dataproc Versions](/dataproc/docs/concepts/versioning/dataproc-versions#supported_cloud_dataproc_versions),
// such as "1.2" (including a subminor version, such as "1.2.29"), or the
// ["preview" version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
// If unspecified, it defaults to the latest version.
string image_version = 1;
// Optional. The properties to set on daemon config files.
@ -434,6 +479,19 @@ message CreateClusterRequest {
// Required. The cluster to create.
Cluster cluster = 2;
// Optional. A unique id used to identify the request. If the server
// receives two [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] requests with the same
// id, then the second request will be ignored and the
// first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the backend
// is returned.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 4;
}
// A request to update a cluster.
@ -451,6 +509,16 @@ message UpdateClusterRequest {
// Required. The changes to the cluster.
Cluster cluster = 3;
// Optional. Timeout for graceful YARN decomissioning. Graceful
// decommissioning allows removing nodes from the cluster without
// interrupting jobs in progress. Timeout specifies how long to wait for jobs
// in progress to finish before forcefully removing nodes (and potentially
// interrupting jobs). Default timeout is 0 (for forceful decommission), and
// the maximum allowed timeout is 1 day.
//
// Only supported on Dataproc image versions 1.2 and higher.
google.protobuf.Duration graceful_decommission_timeout = 6;
// Required. Specifies the path, relative to `Cluster`, of
// the field to update. For example, to change the number of workers
// in a cluster to 5, the `update_mask` parameter would be
@ -499,6 +567,19 @@ message UpdateClusterRequest {
// </tbody>
// </table>
google.protobuf.FieldMask update_mask = 4;
// Optional. A unique id used to identify the request. If the server
// receives two [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] requests with the same
// id, then the second request will be ignored and the
// first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
// backend is returned.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 7;
}
// A request to delete a cluster.
@ -512,6 +593,23 @@ message DeleteClusterRequest {
// Required. The cluster name.
string cluster_name = 2;
// Optional. Specifying the `cluster_uuid` means the RPC should fail
// (with error NOT_FOUND) if cluster with specified UUID does not exist.
string cluster_uuid = 4;
// Optional. A unique id used to identify the request. If the server
// receives two [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] requests with the same
// id, then the second request will be ignored and the
// first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the
// backend is returned.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 5;
}
// Request to get the resource representation for a cluster in a project.
@ -566,10 +664,10 @@ message ListClustersRequest {
// The list of all clusters in a project.
message ListClustersResponse {
// Output-only. The clusters in the project.
// Output only. The clusters in the project.
repeated Cluster clusters = 1;
// Output-only. This token is included in the response if there are more
// Output only. This token is included in the response if there are more
// results to fetch. To fetch additional results, provide this value as the
// `page_token` in a subsequent `ListClustersRequest`.
string next_page_token = 2;
@ -590,7 +688,7 @@ message DiagnoseClusterRequest {
// The location of diagnostic output.
message DiagnoseClusterResults {
// Output-only. The Google Cloud Storage URI of the diagnostic output.
// Output only. The Cloud Storage URI of the diagnostic output.
// The output report is a plain text file with a summary of collected
// diagnostics.
string output_uri = 1;

@ -6,10 +6,12 @@ title: Google Cloud Dataproc API
apis:
- name: google.cloud.dataproc.v1.ClusterController
- name: google.cloud.dataproc.v1.JobController
- name: google.cloud.dataproc.v1.WorkflowTemplateService
types:
- name: google.cloud.dataproc.v1.DiagnoseClusterResults
- name: google.cloud.dataproc.v1.ClusterOperationMetadata
- name: google.cloud.dataproc.v1.WorkflowMetadata
documentation:
summary: 'Manages Hadoop-based clusters and jobs on Google Cloud Platform.'

@ -12,7 +12,6 @@ language_settings:
package_name: Google.Cloud.Dataproc.V1
ruby:
package_name: Google::Cloud::Dataproc::V1
release_level: BETA
php:
package_name: Google\Cloud\Dataproc\V1
nodejs:
@ -136,7 +135,7 @@ interfaces:
request_object_method: true
retry_codes_name: non_idempotent
retry_params_name: default
timeout_millis: 90000
timeout_millis: 30000
long_running:
return_type: google.cloud.dataproc.v1.Cluster
metadata_type: google.cloud.dataproc.v1.ClusterOperationMetadata
@ -180,7 +179,7 @@ interfaces:
request_object_method: true
retry_codes_name: idempotent
retry_params_name: default
timeout_millis: 15000
timeout_millis: 60000
long_running:
return_type: google.protobuf.Empty
metadata_type: google.cloud.dataproc.v1.ClusterOperationMetadata
@ -204,7 +203,7 @@ interfaces:
request_object_method: true
retry_codes_name: idempotent
retry_params_name: default
timeout_millis: 10000
timeout_millis: 60000
- name: ListClusters
flattening:
groups:
@ -224,7 +223,7 @@ interfaces:
resources_field: clusters
retry_codes_name: idempotent
retry_params_name: default
timeout_millis: 10000
timeout_millis: 60000
- name: DiagnoseCluster
flattening:
groups:
@ -239,7 +238,7 @@ interfaces:
request_object_method: true
retry_codes_name: non_idempotent
retry_params_name: default
timeout_millis: 10000
timeout_millis: 60000
long_running:
return_type: google.protobuf.Empty
metadata_type: google.cloud.dataproc.v1.DiagnoseClusterResults
@ -279,7 +278,7 @@ interfaces:
initial_rpc_timeout_millis: 30000
rpc_timeout_multiplier: 1
max_rpc_timeout_millis: 30000
total_timeout_millis: 600000
total_timeout_millis: 900000
# A list of method configurations.
# Common properties:
#
@ -372,9 +371,9 @@ interfaces:
- name: ListJobs
flattening:
groups:
- parameters:
- project_id
- region
- parameters:
- project_id
- region
required_fields:
- project_id
- region
@ -429,4 +428,242 @@ interfaces:
request_object_method: true
retry_codes_name: idempotent
retry_params_name: default
timeout_millis: 30000
# The fully qualified name of the API interface.
- name: google.cloud.dataproc.v1.WorkflowTemplateService
# A list of resource collection configurations.
# Consists of a name_pattern and an entity_name.
# The name_pattern is a pattern to describe the names of the resources of this
# collection, using the platform's conventions for URI patterns. A generator
# may use this to generate methods to compose and decompose such names. The
# pattern should use named placeholders as in `shelves/{shelf}/books/{book}`;
# those will be taken as hints for the parameter names of the generated
# methods. If empty, no name methods are generated.
# The entity_name is the name to be used as a basis for generated methods and
# classes.
collections:
- name_pattern: projects/{project}/regions/{region}
entity_name: region
- name_pattern: projects/{project}/regions/{region}/workflowTemplates/{workflow_template}
entity_name: workflow_template
# Definition for retryable codes.
retry_codes_def:
- name: idempotent
retry_codes:
- UNAVAILABLE
- DEADLINE_EXCEEDED
- name: non_idempotent
retry_codes: []
# Definition for retry/backoff parameters.
retry_params_def:
- name: default
initial_retry_delay_millis: 100
retry_delay_multiplier: 1.3
max_retry_delay_millis: 60000
initial_rpc_timeout_millis: 20000
rpc_timeout_multiplier: 1
max_rpc_timeout_millis: 20000
total_timeout_millis: 600000
# A list of method configurations.
# Common properties:
#
# name - The simple name of the method.
#
# flattening - Specifies the configuration for parameter flattening.
# Describes the parameter groups for which a generator should produce method
# overloads which allow a client to directly pass request message fields as
# method parameters. This information may or may not be used, depending on
# the target language.
# Consists of groups, which each represent a list of parameters to be
# flattened. Each parameter listed must be a field of the request message.
#
# required_fields - Fields that are always required for a request to be
# valid.
#
# request_object_method - Turns on or off the generation of a method whose
# sole parameter is a request object. Not all languages will generate this
# method.
#
# resource_name_treatment - An enum that specifies how to treat the resource
# name formats defined in the field_name_patterns and
# response_field_name_patterns fields.
# UNSET: default value
# NONE: the collection configs will not be used by the generated code.
# VALIDATE: string fields will be validated by the client against the
# specified resource name formats.
# STATIC_TYPES: the client will use generated types for resource names.
#
# page_streaming - Specifies the configuration for paging.
# Describes information for generating a method which transforms a paging
# list RPC into a stream of resources.
# Consists of a request and a response.
# The request specifies request information of the list method. It defines
# which fields match the paging pattern in the request. The request consists
# of a page_size_field and a token_field. The page_size_field is the name of
# the optional field specifying the maximum number of elements to be
# returned in the response. The token_field is the name of the field in the
# request containing the page token.
# The response specifies response information of the list method. It defines
# which fields match the paging pattern in the response. The response
# consists of a token_field and a resources_field. The token_field is the
# name of the field in the response containing the next page token. The
# resources_field is the name of the field in the response containing the
# list of resources belonging to the page.
#
# retry_codes_name - Specifies the configuration for retryable codes. The
# name must be defined in interfaces.retry_codes_def.
#
# retry_params_name - Specifies the configuration for retry/backoff
# parameters. The name must be defined in interfaces.retry_params_def.
#
# field_name_patterns - Maps the field name of the request type to
# entity_name of interfaces.collections.
# Specifies the string pattern that the field must follow.
#
# timeout_millis - Specifies the default timeout for a non-retrying call. If
# the call is retrying, refer to retry_params_name instead.
methods:
- name: CreateWorkflowTemplate
flattening:
groups:
- parameters:
- parent
- template
required_fields:
- parent
- template
request_object_method: true
retry_codes_name: non_idempotent
retry_params_name: default
resource_name_treatment: STATIC_TYPES
field_name_patterns:
parent: region
timeout_millis: 60000
- name: GetWorkflowTemplate
flattening:
groups:
- parameters:
- name
required_fields:
- name
request_object_method: true
retry_codes_name: idempotent
retry_params_name: default
resource_name_treatment: STATIC_TYPES
field_name_patterns:
name: workflow_template
timeout_millis: 60000
- name: InstantiateWorkflowTemplate
flattening:
groups:
- parameters:
- name
- request_id
required_fields:
- name
request_object_method: true
retry_codes_name: non_idempotent
retry_params_name: default
resource_name_treatment: STATIC_TYPES
field_name_patterns:
name: workflow_template
timeout_millis: 60000
long_running:
return_type: google.protobuf.Empty
metadata_type: google.cloud.dataproc.v1.WorkflowMetadata
implements_delete: true
implements_cancel: true
initial_poll_delay_millis: 1000
poll_delay_multiplier: 2
max_poll_delay_millis: 10000
total_poll_timeout_millis: 43200000
- name: InstantiateInlineWorkflowTemplate
flattening:
groups:
- parameters:
- parent
- template
- request_id
required_fields:
- parent
- template
request_object_method: true
retry_codes_name: non_idempotent
retry_params_name: default
field_name_patterns:
parent: region
timeout_millis: 60000
long_running:
return_type: google.protobuf.Empty
metadata_type: google.cloud.dataproc.v1.WorkflowMetadata
implements_delete: true
implements_cancel: true
initial_poll_delay_millis: 1000
poll_delay_multiplier: 2
max_poll_delay_millis: 10000
total_poll_timeout_millis: 43200000
- name: UpdateWorkflowTemplate
flattening:
groups:
- parameters:
- template
required_fields:
- template
request_object_method: false
retry_codes_name: idempotent
retry_params_name: default
timeout_millis: 60000
- name: ListWorkflowTemplates
flattening:
groups:
- parameters:
- parent
required_fields:
- parent
request_object_method: true
page_streaming:
request:
page_size_field: page_size
token_field: page_token
response:
token_field: next_page_token
resources_field: templates
retry_codes_name: idempotent
retry_params_name: default
resource_name_treatment: STATIC_TYPES
field_name_patterns:
parent: region
timeout_millis: 60000
- name: DeleteWorkflowTemplate
flattening:
groups:
- parameters:
- name
required_fields:
- name
request_object_method: true
retry_codes_name: idempotent
retry_params_name: default
resource_name_treatment: STATIC_TYPES
field_name_patterns:
name: workflow_template
timeout_millis: 60000
resource_name_generation:
- message_name: WorkflowTemplate
field_entity_map:
name: workflow_template
- message_name: CreateWorkflowTemplateRequest
field_entity_map:
parent: region
- message_name: GetWorkflowTemplateRequest
field_entity_map:
name: workflow_template
- message_name: InstantiateWorkflowTemplateRequest
field_entity_map:
name: workflow_template
- message_name: ListWorkflowTemplatesRequest
field_entity_map:
parent: region
- message_name: DeleteWorkflowTemplateRequest
field_entity_map:
name: workflow_template

@ -1,4 +1,4 @@
// Copyright 2017 Google Inc.
// Copyright 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
@ -31,22 +32,32 @@ option java_package = "com.google.cloud.dataproc.v1";
service JobController {
// Submits a job to a cluster.
rpc SubmitJob(SubmitJobRequest) returns (Job) {
option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/jobs:submit" body: "*" };
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/jobs:submit"
body: "*"
};
}
// Gets the resource representation for a job in a project.
rpc GetJob(GetJobRequest) returns (Job) {
option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" };
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}"
};
}
// Lists regions/{region}/jobs in a project.
rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) {
option (google.api.http) = { get: "/v1/projects/{project_id}/regions/{region}/jobs" };
option (google.api.http) = {
get: "/v1/projects/{project_id}/regions/{region}/jobs"
};
}
// Updates a job in a project.
rpc UpdateJob(UpdateJobRequest) returns (Job) {
option (google.api.http) = { patch: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" body: "job" };
option (google.api.http) = {
patch: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}"
body: "job"
};
}
// Starts a job cancellation request. To access the job resource
@ -54,13 +65,18 @@ service JobController {
// [regions/{region}/jobs.list](/dataproc/docs/reference/rest/v1/projects.regions.jobs/list) or
// [regions/{region}/jobs.get](/dataproc/docs/reference/rest/v1/projects.regions.jobs/get).
rpc CancelJob(CancelJobRequest) returns (Job) {
option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel" body: "*" };
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel"
body: "*"
};
}
// Deletes the job from the project. If the job is active, the delete fails,
// and the response returns `FAILED_PRECONDITION`.
rpc DeleteJob(DeleteJobRequest) returns (google.protobuf.Empty) {
option (google.api.http) = { delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" };
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}"
};
}
}
@ -362,7 +378,7 @@ message JobPlacement {
// Required. The name of the cluster where the job will be submitted.
string cluster_name = 1;
// Output-only. A cluster UUID generated by the Cloud Dataproc service when
// Output only. A cluster UUID generated by the Cloud Dataproc service when
// the job is submitted.
string cluster_uuid = 2;
}
@ -407,7 +423,9 @@ message JobStatus {
ATTEMPT_FAILURE = 9;
}
// The job substate.
enum Substate {
// The job substate is unknown.
UNSPECIFIED = 0;
// The Job is submitted to the agent.
@ -430,17 +448,17 @@ message JobStatus {
STALE_STATUS = 3;
}
// Output-only. A state message specifying the overall job state.
// Output only. A state message specifying the overall job state.
State state = 1;
// Output-only. Optional job state details, such as an error
// Output only. Optional job state details, such as an error
// description if the state is <code>ERROR</code>.
string details = 2;
// Output-only. The time when this state was entered.
// Output only. The time when this state was entered.
google.protobuf.Timestamp state_start_time = 6;
// Output-only. Additional state information, which includes
// Output only. Additional state information, which includes
// status reported by the agent.
Substate substate = 7;
}
@ -545,25 +563,25 @@ message Job {
SparkSqlJob spark_sql_job = 12;
}
// Output-only. The job status. Additional application-specific
// Output only. The job status. Additional application-specific
// status information may be contained in the <code>type_job</code>
// and <code>yarn_applications</code> fields.
JobStatus status = 8;
// Output-only. The previous job status.
// Output only. The previous job status.
repeated JobStatus status_history = 13;
// Output-only. The collection of YARN applications spun up by this job.
// Output only. The collection of YARN applications spun up by this job.
//
// **Beta** Feature: This report is available for testing purposes only. It may
// be changed before final release.
repeated YarnApplication yarn_applications = 9;
// Output-only. A URI pointing to the location of the stdout of the job's
// Output only. A URI pointing to the location of the stdout of the job's
// driver program.
string driver_output_resource_uri = 17;
// Output-only. If present, the location of miscellaneous control files
// Output only. If present, the location of miscellaneous control files
// which may be used as part of job setup and handling. If not present,
// control files may be placed in the same location as `driver_output_uri`.
string driver_control_files_uri = 15;
@ -578,12 +596,14 @@ message Job {
// Optional. Job scheduling configuration.
JobScheduling scheduling = 20;
// Output only. A UUID that uniquely identifies a job within the project
// over time. This is in contrast to a user-settable reference.job_id that
// may be reused over time.
string job_uuid = 22;
}
// Job scheduling options.
//
// **Beta Feature**: These options are available for testing purposes only.
// They may be changed before final release.
message JobScheduling {
// Optional. Maximum number of times per hour a driver may be restarted as
// a result of driver terminating with non-zero code before job is
@ -607,6 +627,19 @@ message SubmitJobRequest {
// Required. The job resource.
Job job = 2;
// Optional. A unique id used to identify the request. If the server
// receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] requests with the same
// id, then the second request will be ignored and the
// first [Job][google.cloud.dataproc.v1.Job] created and stored in the backend
// is returned.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 4;
}
// A request to get the resource representation for a job in a project.
@ -704,7 +737,7 @@ message UpdateJobRequest {
// A list of jobs in a project.
message ListJobsResponse {
// Output-only. Jobs list.
// Output only. Jobs list.
repeated Job jobs = 1;
// Optional. This token is included in the response if there are more results

@ -1,4 +1,4 @@
// Copyright 2017 Google Inc.
// Copyright 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -11,14 +11,13 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";
option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
@ -44,42 +43,42 @@ message ClusterOperationStatus {
DONE = 3;
}
// Output-only. A message containing the operation state.
// Output only. A message containing the operation state.
State state = 1;
// Output-only. A message containing the detailed operation state.
// Output only. A message containing the detailed operation state.
string inner_state = 2;
// Output-only.A message containing any operation metadata details.
// Output only. A message containing any operation metadata details.
string details = 3;
// Output-only. The time this state was entered.
// Output only. The time this state was entered.
google.protobuf.Timestamp state_start_time = 4;
}
// Metadata describing the operation.
message ClusterOperationMetadata {
// Output-only. Name of the cluster for the operation.
// Output only. Name of the cluster for the operation.
string cluster_name = 7;
// Output-only. Cluster UUID for the operation.
// Output only. Cluster UUID for the operation.
string cluster_uuid = 8;
// Output-only. Current operation status.
// Output only. Current operation status.
ClusterOperationStatus status = 9;
// Output-only. The previous operation status.
// Output only. The previous operation status.
repeated ClusterOperationStatus status_history = 10;
// Output-only. The operation type.
// Output only. The operation type.
string operation_type = 11;
// Output-only. Short description of operation.
// Output only. Short description of operation.
string description = 12;
// Output-only. Labels associated with the operation
// Output only. Labels associated with the operation
map<string, string> labels = 13;
// Output-only. Errors encountered during operation execution.
// Output only. Errors encountered during operation execution.
repeated string warnings = 14;
}

@ -0,0 +1,662 @@
// Copyright 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/cloud/dataproc/v1/clusters.proto";
import "google/cloud/dataproc/v1/jobs.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";
option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc";
option java_multiple_files = true;
option java_outer_classname = "WorkflowTemplatesProto";
option java_package = "com.google.cloud.dataproc.v1";
// The API interface for managing Workflow Templates in the
// Cloud Dataproc API.
service WorkflowTemplateService {
// Creates new workflow template.
rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
body: "template"
additional_bindings {
post: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
body: "template"
}
};
}
// Retrieves the latest workflow template.
//
// Can retrieve previously instantiated template by specifying optional
// version parameter.
rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) returns (WorkflowTemplate) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
additional_bindings {
get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
}
};
}
// Instantiates a template and begins execution.
//
// The returned Operation can be used to track execution of
// workflow by polling
// [operations.get][google.longrunning.Operations.GetOperation].
// The Operation will complete when entire workflow is finished.
//
// The running workflow can be aborted via
// [operations.cancel][google.longrunning.Operations.CancelOperation].
// This will cause any inflight jobs to be cancelled and workflow-owned
// clusters to be deleted.
//
// The [Operation.metadata][google.longrunning.Operation.metadata] will be
// [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
//
// On successful completion,
// [Operation.response][google.longrunning.Operation.response] will be
// [Empty][google.protobuf.Empty].
rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate"
body: "*"
additional_bindings {
post: "/v1/{name=projects/*/regions/*/workflowTemplates/*}:instantiate"
body: "*"
}
};
}
// Instantiates a template and begins execution.
//
// This method is equivalent to executing the sequence
// [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate],
// [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate].
//
// The returned Operation can be used to track execution of
// workflow by polling
// [operations.get][google.longrunning.Operations.GetOperation].
// The Operation will complete when entire workflow is finished.
//
// The running workflow can be aborted via
// [operations.cancel][google.longrunning.Operations.CancelOperation].
// This will cause any inflight jobs to be cancelled and workflow-owned
// clusters to be deleted.
//
// The [Operation.metadata][google.longrunning.Operation.metadata] will be
// [WorkflowMetadata][google.cloud.dataproc.v1.WorkflowMetadata].
//
// On successful completion,
// [Operation.response][google.longrunning.Operation.response] will be
// [Empty][google.protobuf.Empty].
rpc InstantiateInlineWorkflowTemplate(InstantiateInlineWorkflowTemplateRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline"
body: "template"
additional_bindings {
post: "/v1/{parent=projects/*/regions/*}/workflowTemplates:instantiateInline"
body: "template"
}
};
}
// Updates (replaces) workflow template. The updated template
// must contain version that matches the current server version.
rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) returns (WorkflowTemplate) {
option (google.api.http) = {
put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}"
body: "template"
additional_bindings {
put: "/v1/{template.name=projects/*/regions/*/workflowTemplates/*}"
body: "template"
}
};
}
// Lists workflows that match the specified filter in the request.
rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) returns (ListWorkflowTemplatesResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*}/workflowTemplates"
additional_bindings {
get: "/v1/{parent=projects/*/regions/*}/workflowTemplates"
}
};
}
// Deletes a workflow template. It does not cancel in-progress workflows.
rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}"
additional_bindings {
delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}"
}
};
}
}
// A Cloud Dataproc workflow template resource.
message WorkflowTemplate {
// Required. The template id.
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). Cannot begin or end with underscore
// or hyphen. Must consist of between 3 and 50 characters.
string id = 2;
// Output only. The "resource name" of the template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Optional. Used to perform a consistent read-modify-write.
//
// This field should be left blank for a `CreateWorkflowTemplate` request. It
// is required for an `UpdateWorkflowTemplate` request, and must match the
// current server version. A typical update template flow would fetch the
// current template with a `GetWorkflowTemplate` request, which will return
// the current template with the `version` field filled in with the
// current server version. The user updates other fields in the template,
// then returns it as part of the `UpdateWorkflowTemplate` request.
int32 version = 3;
// Output only. The time template was created.
google.protobuf.Timestamp create_time = 4;
// Output only. The time template was last updated.
google.protobuf.Timestamp update_time = 5;
// Optional. The labels to associate with this template. These labels
// will be propagated to all jobs and clusters created by the workflow
// instance.
//
// Label **keys** must contain 1 to 63 characters, and must conform to
// [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
//
// Label **values** may be empty, but, if present, must contain 1 to 63
// characters, and must conform to
// [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
//
// No more than 32 labels can be associated with a template.
map<string, string> labels = 6;
// Required. WorkflowTemplate scheduling information.
WorkflowTemplatePlacement placement = 7;
// Required. The Directed Acyclic Graph of Jobs to submit.
repeated OrderedJob jobs = 8;
// Optional. Template parameters whose values are substituted into the
// template. Values for parameters must be provided when the template is
// instantiated.
repeated TemplateParameter parameters = 9;
}
// Specifies workflow execution target.
//
// Either `managed_cluster` or `cluster_selector` is required.
message WorkflowTemplatePlacement {
// Required. Specifies where workflow executes; either on a managed
// cluster or an existing cluster chosen by labels.
oneof placement {
// Optional. A cluster that is managed by the workflow.
ManagedCluster managed_cluster = 1;
// Optional. A selector that chooses target cluster for jobs based
// on metadata.
//
// The selector is evaluated at the time each job is submitted.
ClusterSelector cluster_selector = 2;
}
}
// Cluster that is managed by the workflow.
message ManagedCluster {
// Required. The cluster name prefix. A unique cluster name will be formed by
// appending a random suffix.
//
// The name must contain only lower-case letters (a-z), numbers (0-9),
// and hyphens (-). Must begin with a letter. Cannot begin or end with
// hyphen. Must consist of between 2 and 35 characters.
string cluster_name = 2;
// Required. The cluster configuration.
ClusterConfig config = 3;
// Optional. The labels to associate with this cluster.
//
// Label keys must be between 1 and 63 characters long, and must conform to
// the following PCRE regular expression:
// [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
//
// Label values must be between 1 and 63 characters long, and must conform to
// the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
//
// No more than 32 labels can be associated with a given cluster.
map<string, string> labels = 4;
}
// A selector that chooses target cluster for jobs based on metadata.
message ClusterSelector {
// Optional. The zone where workflow process executes. This parameter does not
// affect the selection of the cluster.
//
// If unspecified, the zone of the first cluster matching the selector
// is used.
string zone = 1;
// Required. The cluster labels. Cluster must have all labels
// to match.
map<string, string> cluster_labels = 2;
}
// A job executed by the workflow.
message OrderedJob {
// Required. The step id. The id must be unique among all jobs
// within the template.
//
// The step id is used as prefix for job id, as job
// `goog-dataproc-workflow-step-id` label, and in
// [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] field from other
// steps.
//
// The id must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). Cannot begin or end with underscore
// or hyphen. Must consist of between 3 and 50 characters.
string step_id = 1;
// Required. The job definition.
oneof job_type {
// Job is a Hadoop job.
HadoopJob hadoop_job = 2;
// Job is a Spark job.
SparkJob spark_job = 3;
// Job is a Pyspark job.
PySparkJob pyspark_job = 4;
// Job is a Hive job.
HiveJob hive_job = 5;
// Job is a Pig job.
PigJob pig_job = 6;
// Job is a SparkSql job.
SparkSqlJob spark_sql_job = 7;
}
// Optional. The labels to associate with this job.
//
// Label keys must be between 1 and 63 characters long, and must conform to
// the following regular expression:
// [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62}
//
// Label values must be between 1 and 63 characters long, and must conform to
// the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63}
//
// No more than 32 labels can be associated with a given job.
map<string, string> labels = 8;
// Optional. Job scheduling configuration.
JobScheduling scheduling = 9;
// Optional. The optional list of prerequisite job step_ids.
// If not specified, the job will start at the beginning of workflow.
repeated string prerequisite_step_ids = 10;
}
// A configurable parameter that replaces one or more fields in the template.
// Parameterizable fields:
// - Labels
// - File uris
// - Job properties
// - Job arguments
// - Script variables
// - Main class (in HadoopJob and SparkJob)
// - Zone (in ClusterSelector)
message TemplateParameter {
// Required. Parameter name.
// The parameter name is used as the key, and paired with the
// parameter value, which are passed to the template when the template
// is instantiated.
// The name must contain only capital letters (A-Z), numbers (0-9), and
// underscores (_), and must not start with a number. The maximum length is
// 40 characters.
string name = 1;
// Required. Paths to all fields that the parameter replaces.
// A field is allowed to appear in at most one parameter's list of field
// paths.
//
// A field path is similar in syntax to a [google.protobuf.FieldMask][google.protobuf.FieldMask].
// For example, a field path that references the zone field of a workflow
// template's cluster selector would be specified as
// `placement.clusterSelector.zone`.
//
// Also, field paths can reference fields using the following syntax:
//
// * Values in maps can be referenced by key:
// * labels['key']
// * placement.clusterSelector.clusterLabels['key']
// * placement.managedCluster.labels['key']
// * placement.clusterSelector.clusterLabels['key']
// * jobs['step-id'].labels['key']
//
// * Jobs in the jobs list can be referenced by step-id:
// * jobs['step-id'].hadoopJob.mainJarFileUri
// * jobs['step-id'].hiveJob.queryFileUri
// * jobs['step-id'].pySparkJob.mainPythonFileUri
// * jobs['step-id'].hadoopJob.jarFileUris[0]
// * jobs['step-id'].hadoopJob.archiveUris[0]
// * jobs['step-id'].hadoopJob.fileUris[0]
// * jobs['step-id'].pySparkJob.pythonFileUris[0]
//
// * Items in repeated fields can be referenced by a zero-based index:
// * jobs['step-id'].sparkJob.args[0]
//
// * Other examples:
// * jobs['step-id'].hadoopJob.properties['key']
// * jobs['step-id'].hadoopJob.args[0]
// * jobs['step-id'].hiveJob.scriptVariables['key']
// * jobs['step-id'].hadoopJob.mainJarFileUri
// * placement.clusterSelector.zone
//
// It may not be possible to parameterize maps and repeated fields in their
// entirety since only individual map values and individual items in repeated
// fields can be referenced. For example, the following field paths are
// invalid:
//
// - placement.clusterSelector.clusterLabels
// - jobs['step-id'].sparkJob.args
repeated string fields = 2;
// Optional. Brief description of the parameter.
// Must not exceed 1024 characters.
string description = 3;
// Optional. Validation rules to be applied to this parameter's value.
ParameterValidation validation = 4;
}
// Configuration for parameter validation.
message ParameterValidation {
// Required. The type of validation to be performed.
oneof validation_type {
// Validation based on regular expressions.
RegexValidation regex = 1;
// Validation based on a list of allowed values.
ValueValidation values = 2;
}
}
// Validation based on regular expressions.
message RegexValidation {
// Required. RE2 regular expressions used to validate the parameter's value.
// The value must match the regex in its entirety (substring
// matches are not sufficient).
repeated string regexes = 1;
}
// Validation based on a list of allowed values.
message ValueValidation {
// Required. List of allowed values for the parameter.
repeated string values = 1;
}
// A Cloud Dataproc workflow template resource.
message WorkflowMetadata {
// The operation state.
enum State {
// Unused.
UNKNOWN = 0;
// The operation has been created.
PENDING = 1;
// The operation is running.
RUNNING = 2;
// The operation is done; either cancelled or completed.
DONE = 3;
}
// Output only. The "resource name" of the template.
string template = 1;
// Output only. The version of template at the time of
// workflow instantiation.
int32 version = 2;
// Output only. The create cluster operation metadata.
ClusterOperation create_cluster = 3;
// Output only. The workflow graph.
WorkflowGraph graph = 4;
// Output only. The delete cluster operation metadata.
ClusterOperation delete_cluster = 5;
// Output only. The workflow state.
State state = 6;
// Output only. The name of the target cluster.
string cluster_name = 7;
// Map from parameter names to values that were used for those parameters.
map<string, string> parameters = 8;
// Output only. Workflow start time.
google.protobuf.Timestamp start_time = 9;
// Output only. Workflow end time.
google.protobuf.Timestamp end_time = 10;
// Output only. The UUID of target cluster.
string cluster_uuid = 11;
}
// The cluster operation triggered by a workflow.
message ClusterOperation {
// Output only. The id of the cluster operation.
string operation_id = 1;
// Output only. Error, if operation failed.
string error = 2;
// Output only. Indicates the operation is done.
bool done = 3;
}
// The workflow graph.
message WorkflowGraph {
// Output only. The workflow nodes.
repeated WorkflowNode nodes = 1;
}
// The workflow node.
message WorkflowNode {
// The workflow node state.
enum NodeState {
// State is unspecified.
NODE_STATE_UNSPECIFIED = 0;
// The node is awaiting prerequisite node to finish.
BLOCKED = 1;
// The node is runnable but not running.
RUNNABLE = 2;
// The node is running.
RUNNING = 3;
// The node completed successfully.
COMPLETED = 4;
// The node failed. A node can be marked FAILED because
// its ancestor or peer failed.
FAILED = 5;
}
// Output only. The name of the node.
string step_id = 1;
// Output only. Node's prerequisite nodes.
repeated string prerequisite_step_ids = 2;
// Output only. The job id; populated after the node enters RUNNING state.
string job_id = 3;
// Output only. The node state.
NodeState state = 5;
// Output only. The error detail.
string error = 6;
}
// A request to create a workflow template.
message CreateWorkflowTemplateRequest {
// Required. The "resource name" of the region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Required. The Dataproc workflow template to create.
WorkflowTemplate template = 2;
}
// A request to fetch a workflow template.
message GetWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Optional. The version of workflow template to retrieve. Only previously
// instatiated versions can be retrieved.
//
// If unspecified, retrieves the current version.
int32 version = 2;
}
// A request to instantiate a workflow template.
message InstantiateWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Optional. The version of workflow template to instantiate. If specified,
// the workflow will be instantiated only if the current version of
// the workflow template has the supplied version.
//
// This option cannot be used to instantiate a previous version of
// workflow template.
int32 version = 2;
// Optional. A tag that prevents multiple concurrent workflow
// instances with the same tag from running. This mitigates risk of
// concurrent instances started due to retries.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The tag must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 5;
// Optional. Map from parameter names to values that should be used for those
// parameters. Values may not exceed 100 characters.
map<string, string> parameters = 6;
}
// A request to instantiate an inline workflow template.
message InstantiateInlineWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Required. The workflow template to instantiate.
WorkflowTemplate template = 2;
// Optional. A tag that prevents multiple concurrent workflow
// instances with the same tag from running. This mitigates risk of
// concurrent instances started due to retries.
//
// It is recommended to always set this value to a
// [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier).
//
// The tag must contain only letters (a-z, A-Z), numbers (0-9),
// underscores (_), and hyphens (-). The maximum length is 40 characters.
string request_id = 3;
}
// A request to update a workflow template.
message UpdateWorkflowTemplateRequest {
// Required. The updated workflow template.
//
// The `template.version` field must match the current version.
WorkflowTemplate template = 1;
}
// A request to list workflow templates in a project.
message ListWorkflowTemplatesRequest {
// Required. The "resource name" of the region, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}`
string parent = 1;
// Optional. The maximum number of results to return in each response.
int32 page_size = 2;
// Optional. The page token, returned by a previous call, to request the
// next page of results.
string page_token = 3;
}
// A response to a request to list workflow templates in a project.
message ListWorkflowTemplatesResponse {
// Output only. WorkflowTemplates list.
repeated WorkflowTemplate templates = 1;
// Output only. This token is included in the response if there are more
// results to fetch. To fetch additional results, provide this value as the
// page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>.
string next_page_token = 2;
}
// A request to delete a workflow template.
//
// Currently started workflows will remain running.
message DeleteWorkflowTemplateRequest {
// Required. The "resource name" of the workflow template, as described
// in https://cloud.google.com/apis/design/resource_names of the form
// `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}`
string name = 1;
// Optional. The version of workflow template to delete. If specified,
// will only delete the template if the current server version matches
// specified version.
int32 version = 2;
}
Loading…
Cancel
Save