googleapis/google/devtools/testing/v1/test_execution.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.devtools.testing.v1;

import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";

option go_package = "google.golang.org/genproto/googleapis/devtools/testing/v1;testing";
option java_multiple_files = true;
option java_outer_classname = "TestExecutionProto";
option java_package = "com.google.devtools.testing.v1";

// A service for requesting test executions and querying their status.
//
// This service is part of Firebase Test Lab. To learn about how to use the
// product, and how to integrate it with your system,
// visit https://firebase.google.com/docs/test-lab.
//
// Each test execution will wait for available capacity. It will then be
// invoked as described. The test may be invoked multiple times if an
// infrastructure failure is detected. Results and other files generated by
// the test will be stored in an external storage system.
//
// The TestExecutionService models this behavior using two resource types:
//
// - TestMatrix: a group of one or more TestExecutions, built by taking a
//   product of values over a pre-defined set of axes. In the case of Android
//   Tests, for example, device model and OS version are two axes of the matrix.
//
// - TestExecution: a single execution of one or more test targets on a
//   single device. These are created automatically when a TestMatrix is
//   created.
//
// This service returns any error codes from the canonical error space (i.e.
// google.rpc.Code). The errors which may be returned are specified on each
// method. In addition, any method may return UNAVAILABLE or INTERNAL.
service TestExecutionService {
  option (google.api.default_host) = "testing.googleapis.com";
  option (google.api.oauth_scopes) =
      "https://www.googleapis.com/auth/cloud-platform,"
      "https://www.googleapis.com/auth/cloud-platform.read-only";

  // Creates and runs a matrix of tests according to the given specifications.
  // Unsupported environments will be returned in the state UNSUPPORTED.
  // A test matrix is limited to use at most 2000 devices in parallel.
  //
  // The returned matrix will not yet contain the executions that will be
  // created for this matrix. Execution creation happens later on and will
  // require a call to GetTestMatrix.
  //
  // May return any of the following canonical error codes:
  //
  // - PERMISSION_DENIED - if the user is not authorized to write to project
  // - INVALID_ARGUMENT - if the request is malformed or if the matrix tries
  //                      to use too many simultaneous devices.
  rpc CreateTestMatrix(CreateTestMatrixRequest) returns (TestMatrix) {
    option (google.api.http) = {
      post: "/v1/projects/{project_id}/testMatrices"
      body: "test_matrix"
    };
  }

  // Checks the status of a test matrix and the executions once they
  // are created.
  //
  // The test matrix will contain the list of test executions to run if and only
  // if the resultStorage.toolResultsExecution fields have been populated.
  //
  // Note: Flaky test executions may be added to the matrix at a later stage.
  //
  // May return any of the following canonical error codes:
  //
  // - PERMISSION_DENIED - if the user is not authorized to read project
  // - INVALID_ARGUMENT - if the request is malformed
  // - NOT_FOUND - if the Test Matrix does not exist
  rpc GetTestMatrix(GetTestMatrixRequest) returns (TestMatrix) {
    option (google.api.http) = {
      get: "/v1/projects/{project_id}/testMatrices/{test_matrix_id}"
    };
  }

  // Cancels unfinished test executions in a test matrix.
  // This call returns immediately and cancellation proceeds asynchronously.
  // If the matrix is already final, this operation will have no effect.
  //
  // May return any of the following canonical error codes:
  //
  // - PERMISSION_DENIED - if the user is not authorized to read project
  // - INVALID_ARGUMENT - if the request is malformed
  // - NOT_FOUND - if the Test Matrix does not exist
  rpc CancelTestMatrix(CancelTestMatrixRequest)
      returns (CancelTestMatrixResponse) {
    option (google.api.http) = {
      post: "/v1/projects/{project_id}/testMatrices/{test_matrix_id}:cancel"
    };
  }
}

// TestMatrix captures all details about a test. It contains the environment
// configuration, test specification, test executions and overall state and
// outcome.
message TestMatrix {
  // Output only. Unique id set by the service.
  string test_matrix_id = 1;

  // The cloud project that owns the test matrix.
  string project_id = 7;

  // Information about the client which invoked the test.
  ClientInfo client_info = 10;

  // Required. How to run the test.
  TestSpecification test_specification = 3;

  // Required. The devices the tests are being executed on.
  EnvironmentMatrix environment_matrix = 4;

  // Output only. The list of test executions that the service creates for
  // this matrix.
  repeated TestExecution test_executions = 5;

  // Required. Where the results for the matrix are written.
  ResultStorage result_storage = 6;

  // Output only. Indicates the current progress of the test matrix.
  TestState state = 8;

  // Output only. The time this test matrix was initially created.
  google.protobuf.Timestamp timestamp = 9;

  // Output only. Describes why the matrix is considered invalid.
  // Only useful for matrices in the INVALID state.
  InvalidMatrixDetails invalid_matrix_details = 11;

  // The number of times a TestExecution should be re-attempted if one or more
  // of its test cases fail for any reason.
  // The maximum number of reruns allowed is 10.
  //
  // Default is 0, which implies no reruns.
  int32 flaky_test_attempts = 13;

  // Output Only. The overall outcome of the test.
  // Only set when the test matrix state is FINISHED.
  OutcomeSummary outcome_summary = 14;

  // If true, only a single attempt at most will be made to run each
  // execution/shard in the matrix. Flaky test attempts are not affected.
  //
  // Normally, 2 or more attempts are made if a potential infrastructure issue
  // is detected.
  //
  // This feature is for latency sensitive workloads. The incidence of
  // execution failures may be significantly greater for fail-fast matrices
  // and support is more limited because of that expectation.
  bool fail_fast = 17;
}

// A single test executed in a single environment.
message TestExecution {
  // Output only. Unique id set by the service.
  string id = 1;

  // Output only. Id of the containing TestMatrix.
  string matrix_id = 9;

  // Output only. The cloud project that owns the test execution.
  string project_id = 10;

  // Output only. How to run the test.
  TestSpecification test_specification = 3;

  // Output only. Details about the shard.
  Shard shard = 12;

  // Output only. How the host machine(s) are configured.
  Environment environment = 4;

  // Output only. Indicates the current progress of the test execution
  // (e.g., FINISHED).
  TestState state = 5;

  // Output only. Where the results for this execution are written.
  ToolResultsStep tool_results_step = 11;

  // Output only. The time this test execution was initially created.
  google.protobuf.Timestamp timestamp = 7;

  // Output only. Additional details about the running test.
  TestDetails test_details = 8;
}

// A description of how to run the test.
message TestSpecification {
  // Max time a test execution is allowed to run before it is
  // automatically cancelled.
  // The default value is 5 min.
  google.protobuf.Duration test_timeout = 1;

  // Test setup requirements.
  oneof setup {
    // Test setup requirements for Android e.g. files to install, bootstrap
    // scripts.
    TestSetup test_setup = 6;

    // Test setup requirements for iOS.
    IosTestSetup ios_test_setup = 14;
  }

  // Required. The type of test to run.
  oneof test {
    // An Android instrumentation test.
    AndroidInstrumentationTest android_instrumentation_test = 2;

    // An Android robo test.
    AndroidRoboTest android_robo_test = 3;

    // An Android Application with a Test Loop.
    AndroidTestLoop android_test_loop = 9;

    // An iOS XCTest, via an .xctestrun file.
    IosXcTest ios_xc_test = 13;

    // An iOS application with a test loop.
    IosTestLoop ios_test_loop = 15;
  }

  // Disables video recording. May reduce test latency.
  bool disable_video_recording = 10;

  // Disables performance metrics recording. May reduce test latency.
  bool disable_performance_metrics = 11;
}

message SystraceSetup {
  // Systrace duration in seconds.
  // Should be between 1 and 30 seconds. 0 disables systrace.
  int32 duration_seconds = 1 [deprecated = true];
}

// A description of how to set up the Android device prior to running the test.
message TestSetup {
  // List of files to push to the device before starting the test.
  repeated DeviceFile files_to_push = 1;

  // List of directories on the device to upload to GCS at the end of the test;
  // they must be absolute paths under /sdcard, /storage or /data/local/tmp.
  // Path names are restricted to characters a-z A-Z 0-9 _ - . + and /
  //
  // Note: The paths /sdcard and /data will be made available and treated as
  // implicit path substitutions. E.g. if /sdcard on a particular device does
  // not map to external storage, the system will replace it with the external
  // storage path prefix for that device.
  repeated string directories_to_pull = 2;

  // APKs to install in addition to those being directly tested.
  // Currently capped at 100.
  repeated Apk additional_apks = 3;

  // The device will be logged in on this account for the duration of the test.
  Account account = 4;

  // The network traffic profile used for running the test.
  // Available network profiles can be queried by using the
  // NETWORK_CONFIGURATION environment type when calling
  // TestEnvironmentDiscoveryService.GetTestEnvironmentCatalog.
  string network_profile = 5;

  // Environment variables to set for the test (only applicable for
  // instrumentation tests).
  repeated EnvironmentVariable environment_variables = 6;

  // Systrace configuration for the run.
  // Deprecated: Systrace used Python 2 which was sunsetted on 2020-01-01.
  // Systrace is no longer supported in the Cloud Testing API, and no Systrace
  // file will be provided in the results.
  SystraceSetup systrace = 9 [deprecated = true];

  // Whether to prevent all runtime permissions to be granted at app install
  bool dont_autogrant_permissions = 23;
}

// A description of how to set up an iOS device prior to running the test.
message IosTestSetup {
  // The network traffic profile used for running the test.
  // Available network profiles can be queried by using the
  // NETWORK_CONFIGURATION environment type when calling
  // TestEnvironmentDiscoveryService.GetTestEnvironmentCatalog.
  string network_profile = 1;

  // iOS apps to install in addition to those being directly tested.
  repeated FileReference additional_ipas = 2;

  // List of files to push to the device before starting the test.
  repeated IosDeviceFile push_files = 3;

  // List of directories on the device to upload to Cloud Storage at the end of
  // the test.
  //
  // Directories should either be in a shared directory (such as
  // /private/var/mobile/Media) or within an accessible directory inside the
  // app's filesystem (such as /Documents) by specifying the bundle ID.
  repeated IosDeviceFile pull_directories = 4;
}

// A key-value pair passed as an environment variable to the test.
message EnvironmentVariable {
  // Key for the environment variable.
  string key = 1;

  // Value for the environment variable.
  string value = 2;
}

// Identifies an account and how to log into it.
message Account {
  // Required. The type of account, based what it's for (e.g. Google) and what
  // its login mechanism is (e.g. username and password).
  oneof account_type {
    // An automatic google login account.
    GoogleAuto google_auto = 1;
  }
}

// Enables automatic Google account login.
// If set, the service automatically generates a Google test account and adds
// it to the device, before executing the test. Note that test accounts might be
// reused.
// Many applications show their full set of functionalities when an account is
// present on the device. Logging into the device with these generated accounts
// allows testing more functionalities.
message GoogleAuto {}

// An Android package file to install.
message Apk {
  // The path to an APK to be installed on the device before the test begins.
  FileReference location = 1;

  // The java package for the APK to be installed.
  // Value is determined by examining the application's manifest.
  string package_name = 2;
}

// An Android App Bundle file format, containing a BundleConfig.pb file,
// a base module directory, zero or more dynamic feature module directories.
// <p>See https://developer.android.com/guide/app-bundle/build for guidance on
// building App Bundles.
message AppBundle {
  // Required. Bundle location information.
  oneof bundle {
    // .aab file representing the app bundle under test.
    FileReference bundle_location = 1;
  }
}

// A single device file description.
message DeviceFile {
  // Required.
  oneof device_file {
    // A reference to an opaque binary blob file.
    ObbFile obb_file = 1;

    // A reference to a regular file.
    RegularFile regular_file = 2;
  }
}

// An opaque binary blob file to install on the device before the test starts.
message ObbFile {
  // Required. OBB file name which must conform to the format as specified by
  // Android
  // e.g. [main|patch].0300110.com.example.android.obb
  // which will be installed into
  //   \<shared-storage\>/Android/obb/\<package-name\>/
  // on the device.
  string obb_file_name = 1;

  // Required. Opaque Binary Blob (OBB) file(s) to install on the device.
  FileReference obb = 2;
}

// A file or directory to install on the device before the test starts.
message RegularFile {
  // Required. The source file.
  FileReference content = 1;

  // Required. Where to put the content on the device. Must be an absolute,
  // allowlisted path. If the file exists, it will be replaced.
  // The following device-side directories and any of their subdirectories are
  // allowlisted:
  // <p>${EXTERNAL_STORAGE}, /sdcard, or /storage</p>
  // <p>${ANDROID_DATA}/local/tmp, or /data/local/tmp</p>
  // <p>Specifying a path outside of these directory trees is invalid.
  //
  // <p> The paths /sdcard and /data will be made available and treated as
  // implicit path substitutions. E.g. if /sdcard on a particular device does
  // not map to external storage, the system will replace it with the external
  // storage path prefix for that device and copy the file there.
  //
  // <p> It is strongly advised to use the <a href=
  // "http://developer.android.com/reference/android/os/Environment.html">
  // Environment API</a> in app and test code to access files on the device in a
  // portable way.
  string device_path = 2;
}

// A file or directory to install on the device before the test starts.
message IosDeviceFile {
  // The source file
  FileReference content = 1;

  // The bundle id of the app where this file lives.
  //
  // iOS apps sandbox their own filesystem, so app files must specify which app
  // installed on the device.
  string bundle_id = 2;

  // Location of the file on the device, inside the app's sandboxed filesystem
  string device_path = 3;
}

// A test of an Android Application with a Test Loop.
// The intent \<intent-name\> will be implicitly added, since Games is the only
// user of this api, for the time being.
message AndroidTestLoop {
  // Required. The Android package to test.
  oneof app_under_test {
    // The APK for the application under test.
    FileReference app_apk = 1;

    // A multi-apk app bundle for the application under test.
    AppBundle app_bundle = 5;
  }

  // The java package for the application under test.
  // The default is determined by examining the application's manifest.
  string app_package_id = 2;

  // The list of scenarios that should be run during the test.
  // The default is all test loops, derived from the application's
  // manifest.
  repeated int32 scenarios = 3;

  // The list of scenario labels that should be run during the test.
  // The scenario labels should map to labels defined in the application's
  // manifest. For example, player_experience and
  // com.google.test.loops.player_experience add all of the loops labeled in the
  // manifest with the com.google.test.loops.player_experience name to the
  // execution.
  // Scenarios can also be specified in the scenarios field.
  repeated string scenario_labels = 4;
}

// A test of an iOS application that uses the XCTest framework.
// Xcode supports the option to "build for testing", which generates an
// .xctestrun file that contains a test specification (arguments, test methods,
// etc). This test type accepts a zip file containing the .xctestrun file and
// the corresponding contents of the Build/Products directory that contains all
// the binaries needed to run the tests.
message IosXcTest {
  // Required. The .zip containing the .xctestrun file and the contents of the
  // DerivedData/Build/Products directory.
  // The .xctestrun file in this zip is ignored if the xctestrun field is
  // specified.
  FileReference tests_zip = 1;

  // An .xctestrun file that will override the .xctestrun file in the
  // tests zip. Because the .xctestrun file contains environment variables along
  // with test methods to run and/or ignore, this can be useful for sharding
  // tests. Default is taken from the tests zip.
  FileReference xctestrun = 2;

  // The Xcode version that should be used for the test.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  // Defaults to the latest Xcode version Firebase Test Lab supports.
  string xcode_version = 3;

  // Output only. The bundle id for the application under test.
  string app_bundle_id = 4;

  // The option to test special app entitlements. Setting this would re-sign the
  // app having special entitlements with an explicit application-identifier.
  // Currently supports testing aps-environment entitlement.
  bool test_special_entitlements = 6;
}

// A test of an iOS application that implements one or more game loop scenarios.
// This test type accepts an archived application (.ipa file) and a list of
// integer scenarios that will be executed on the app sequentially.
message IosTestLoop {
  // Required. The .ipa of the application to test.
  FileReference app_ipa = 1;

  // The list of scenarios that should be run during the test. Defaults to the
  // single scenario 0 if unspecified.
  repeated int32 scenarios = 2;

  // Output only. The bundle id for the application under test.
  string app_bundle_id = 3;
}

// A test of an Android application that can control an Android component
// independently of its normal lifecycle.
// Android instrumentation tests run an application APK and test APK inside the
// same process on a virtual or physical AndroidDevice.  They also specify
// a test runner class, such as com.google.GoogleTestRunner, which can vary
// on the specific instrumentation framework chosen.
//
// See <https://developer.android.com/training/testing/fundamentals> for
// more information on types of Android tests.
message AndroidInstrumentationTest {
  // Required.
  oneof app_under_test {
    // The APK for the application under test.
    FileReference app_apk = 1;

    // A multi-apk app bundle for the application under test.
    AppBundle app_bundle = 8;
  }

  // Required. The APK containing the test code to be executed.
  FileReference test_apk = 2;

  // The java package for the application under test.
  // The default value is determined by examining the application's manifest.
  string app_package_id = 3;

  // The java package for the test to be executed.
  // The default value is determined by examining the application's manifest.
  string test_package_id = 4;

  // The InstrumentationTestRunner class.
  // The default value is determined by examining the application's manifest.
  string test_runner_class = 5;

  // Each target must be fully qualified with the package name or class name,
  // in one of these formats:
  //
  //  - "package package_name"
  //  - "class package_name.class_name"
  //  - "class package_name.class_name#method_name"
  //
  // If empty, all targets in the module will be run.
  repeated string test_targets = 6;

  // The option of whether running each test within its own invocation of
  // instrumentation with Android Test Orchestrator or not.
  // ** Orchestrator is only compatible with AndroidJUnitRunner version 1.1 or
  // higher! **
  // Orchestrator offers the following benefits:
  //
  //  - No shared state
  //  - Crashes are isolated
  //  - Logs are scoped per test
  //
  // See
  // <https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator>
  // for more information about Android Test Orchestrator.
  //
  // If not set, the test will be run without the orchestrator.
  OrchestratorOption orchestrator_option = 7;

  // The option to run tests in multiple shards in parallel.
  ShardingOption sharding_option = 9;
}

// Specifies how to execute the test.
enum OrchestratorOption {
  // Default value: the server will choose the mode. Currently implies that
  // the test will run without the orchestrator. In the future,
  // all instrumentation tests will be run with the orchestrator.
  // Using the orchestrator is highly encouraged because of all the benefits it
  // offers.
  ORCHESTRATOR_OPTION_UNSPECIFIED = 0;

  // Run test using orchestrator.
  // ** Only compatible with AndroidJUnitRunner version 1.1 or higher! **
  // Recommended.
  USE_ORCHESTRATOR = 1;

  // Run test without using orchestrator.
  DO_NOT_USE_ORCHESTRATOR = 2;
}

// A test of an android application that explores the application on a virtual
// or physical Android Device, finding culprits and crashes as it goes.
message AndroidRoboTest {
  // Required.
  oneof app_under_test {
    // The APK for the application under test.
    FileReference app_apk = 1;

    // A multi-apk app bundle for the application under test.
    AppBundle app_bundle = 16;
  }

  // The java package for the application under test.
  // The default value is determined by examining the application's manifest.
  string app_package_id = 2;

  // The initial activity that should be used to start the app.
  string app_initial_activity = 3;

  // The max depth of the traversal stack Robo can explore. Needs to be at least
  // 2 to make Robo explore the app beyond the first activity.
  // Default is 50.
  int32 max_depth = 7 [deprecated = true];

  // The max number of steps Robo can execute.
  // Default is no limit.
  int32 max_steps = 8 [deprecated = true];

  // A set of directives Robo should apply during the crawl.
  // This allows users to customize the crawl. For example, the username and
  // password for a test account can be provided.
  repeated RoboDirective robo_directives = 11;

  // The mode in which Robo should run. Most clients should allow the server to
  // populate this field automatically.
  RoboMode robo_mode = 14;

  // A JSON file with a sequence of actions Robo should perform as a prologue
  // for the crawl.
  FileReference robo_script = 13;

  // The intents used to launch the app for the crawl.
  // If none are provided, then the main launcher activity is launched.
  // If some are provided, then only those provided are launched (the main
  // launcher activity must be provided explicitly).
  repeated RoboStartingIntent starting_intents = 15;
}

// The mode in which Robo should run.
enum RoboMode {
  // This means that the server should choose the mode.
  // Recommended.
  ROBO_MODE_UNSPECIFIED = 0;

  // Runs Robo in UIAutomator-only mode without app resigning
  ROBO_VERSION_1 = 1;

  // Runs Robo in standard Espresso with UIAutomator fallback
  ROBO_VERSION_2 = 2;
}

// Directs Robo to interact with a specific UI element if it is encountered
// during the crawl. Currently, Robo can perform text entry or element click.
message RoboDirective {
  // Required. The android resource name of the target UI element.
  // For example,
  //    in Java: R.string.foo
  //    in xml: @string/foo
  // Only the "foo" part is needed.
  // Reference doc:
  // https://developer.android.com/guide/topics/resources/accessing-resources.html
  string resource_name = 1;

  // The text that Robo is directed to set. If left empty, the directive will be
  // treated as a CLICK on the element matching the resource_name.
  string input_text = 2;

  // Required. The type of action that Robo should perform on the specified
  // element.
  RoboActionType action_type = 3;
}

// Actions which Robo can perform on UI elements.
enum RoboActionType {
  // DO NOT USE. For proto versioning only.
  ACTION_TYPE_UNSPECIFIED = 0;

  // Direct Robo to click on the specified element. No-op if specified element
  // is not clickable.
  SINGLE_CLICK = 1;

  // Direct Robo to enter text on the specified element. No-op if specified
  // element is not enabled or does not allow text entry.
  ENTER_TEXT = 2;

  // Direct Robo to ignore interactions with a specific element.
  IGNORE = 3;
}

// Message for specifying the start activities to crawl.
message RoboStartingIntent {
  // Required. Intent details to start an activity.
  oneof starting_intent {
    // An intent that starts the main launcher activity.
    LauncherActivityIntent launcher_activity = 1;

    // An intent that starts an activity with specific details.
    StartActivityIntent start_activity = 2;
  }

  // Timeout in seconds for each intent.
  google.protobuf.Duration timeout = 3;
}

// Specifies an intent that starts the main launcher activity.
message LauncherActivityIntent {}

// A starting intent specified by an action, uri, and categories.
message StartActivityIntent {
  // Action name.
  // Required for START_ACTIVITY.
  string action = 2;

  // URI for the action.
  string uri = 3;

  // Intent categories to set on the intent.
  repeated string categories = 4;
}

// The matrix of environments in which the test is to be executed.
message EnvironmentMatrix {
  // Required. The environment matrix.
  oneof environment_matrix {
    // A matrix of Android devices.
    AndroidMatrix android_matrix = 1;

    // A list of Android devices; the test will be run only on the specified
    // devices.
    AndroidDeviceList android_device_list = 2;

    // A list of iOS devices.
    IosDeviceList ios_device_list = 3;
  }
}

// A list of Android device configurations in which the test is to be executed.
message AndroidDeviceList {
  // Required. A list of Android devices.
  repeated AndroidDevice android_devices = 1;
}

// A list of iOS device configurations in which the test is to be executed.
message IosDeviceList {
  // Required. A list of iOS devices.
  repeated IosDevice ios_devices = 1;
}

// A set of Android device configuration permutations is defined by the
// the cross-product of the given axes. Internally, the given AndroidMatrix
// will be expanded into a set of AndroidDevices.
//
// Only supported permutations will be instantiated.  Invalid permutations
// (e.g., incompatible models/versions) are ignored.
message AndroidMatrix {
  // Required. The ids of the set of Android device to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  repeated string android_model_ids = 1;

  // Required. The ids of the set of Android OS version to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  repeated string android_version_ids = 2;

  // Required. The set of locales the test device will enable for testing.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  repeated string locales = 3;

  // Required. The set of orientations to test with.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  repeated string orientations = 4;
}

// Information about the client which invoked the test.
message ClientInfo {
  // Required. Client name, such as gcloud.
  string name = 1;

  // The list of detailed information about client.
  repeated ClientInfoDetail client_info_details = 2;
}

// Key-value pair of detailed information about the client which invoked the
// test. Examples: {'Version', '1.0'}, {'Release Track', 'BETA'}.
message ClientInfoDetail {
  // Required. The key of detailed client information.
  string key = 1;

  // Required. The value of detailed client information.
  string value = 2;
}

// Locations where the results of running the test are stored.
message ResultStorage {
  // Required.
  GoogleCloudStorage google_cloud_storage = 1;

  // The tool results history that contains the tool results execution that
  // results are written to.
  //
  // If not provided, the service will choose an appropriate value.
  ToolResultsHistory tool_results_history = 5;

  // Output only. The tool results execution that results are written to.
  ToolResultsExecution tool_results_execution = 6;

  // Output only. URL to the results in the Firebase Web Console.
  string results_url = 7;
}

// Represents a tool results history resource.
message ToolResultsHistory {
  // Required. The cloud project that owns the tool results history.
  string project_id = 1;

  // Required. A tool results history ID.
  string history_id = 2;
}

// Represents a tool results execution resource.
//
// This has the results of a TestMatrix.
message ToolResultsExecution {
  // Output only. The cloud project that owns the tool results execution.
  string project_id = 1;

  // Output only. A tool results history ID.
  string history_id = 2;

  // Output only. A tool results execution ID.
  string execution_id = 3;
}

// Represents a tool results step resource.
//
// This has the results of a TestExecution.
message ToolResultsStep {
  // Output only. The cloud project that owns the tool results step.
  string project_id = 1;

  // Output only. A tool results history ID.
  string history_id = 2;

  // Output only. A tool results execution ID.
  string execution_id = 3;

  // Output only. A tool results step ID.
  string step_id = 4;
}

// A storage location within Google cloud storage (GCS).
message GoogleCloudStorage {
  // Required. The path to a directory in GCS that will
  // eventually contain the results for this test.
  // The requesting user must have write access on the bucket in the supplied
  // path.
  string gcs_path = 1;
}

// A reference to a file, used for user inputs.
message FileReference {
  // Required. The file reference.
  oneof file {
    // A path to a file in Google Cloud Storage.
    // Example: gs://build-app-1414623860166/app%40debug-unaligned.apk
    // These paths are expected to be url encoded (percent encoding)
    string gcs_path = 1;
  }
}

// The environment in which the test is run.
message Environment {
  // Required. The environment.
  oneof environment {
    // An Android device which must be used with an Android test.
    AndroidDevice android_device = 1;

    // An iOS device which must be used with an iOS test.
    IosDevice ios_device = 2;
  }
}

// A single Android device.
message AndroidDevice {
  // Required. The id of the Android device to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string android_model_id = 1;

  // Required. The id of the Android OS version to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string android_version_id = 2;

  // Required. The locale the test device used for testing.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string locale = 3;

  // Required. How the device is oriented during the test.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string orientation = 4;
}

// A single iOS device.
message IosDevice {
  // Required. The id of the iOS device to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string ios_model_id = 1;

  // Required. The id of the iOS major software version to be used.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string ios_version_id = 2;

  // Required. The locale the test device used for testing.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string locale = 3;

  // Required. How the device is oriented during the test.
  // Use the TestEnvironmentDiscoveryService to get supported options.
  string orientation = 4;
}

// Additional details about the progress of the running test.
message TestDetails {
  // Output only. Human-readable, detailed descriptions of the test's progress.
  // For example: "Provisioning a device", "Starting Test".
  //
  // During the course of execution new data may be appended
  // to the end of progress_messages.
  repeated string progress_messages = 3;

  // Output only. If the TestState is ERROR, then this string will contain
  // human-readable details about the error.
  string error_message = 4;
}

// Details behind an invalid request.
message InvalidRequestDetail {
  // Possible invalid request reasons.
  enum Reason {
    // No reason has been specified - the default.
    REASON_UNSPECIFIED = 0;

    // The request is not valid.
    REQUEST_INVALID = 1;

    // One or more of the resources specified in the request is too large.
    RESOURCE_TOO_BIG = 2;

    // One or more resources specified in the request cannot be found.
    RESOURCE_NOT_FOUND = 3;

    // This request is not (currently) supported.
    UNSUPPORTED = 4;

    // This request is not currently implemented.
    NOT_IMPLEMENTED = 5;
  }

  // The reason behind the error.
  Reason reason = 1;
}

// The detailed reason that a Matrix was deemed INVALID.
enum InvalidMatrixDetails {
  // Do not use. For proto versioning only.
  INVALID_MATRIX_DETAILS_UNSPECIFIED = 0;

  // The matrix is INVALID, but there are no further details available.
  DETAILS_UNAVAILABLE = 1;

  // The input app APK could not be parsed.
  MALFORMED_APK = 2;

  // The input test APK could not be parsed.
  MALFORMED_TEST_APK = 3;

  // The AndroidManifest.xml could not be found.
  NO_MANIFEST = 4;

  // The APK manifest does not declare a package name.
  NO_PACKAGE_NAME = 5;

  // The APK application ID (aka package name) is invalid.
  // See also
  // https://developer.android.com/studio/build/application-id
  INVALID_PACKAGE_NAME = 31;

  // The test package and app package are the same.
  TEST_SAME_AS_APP = 6;

  // The test apk does not declare an instrumentation.
  NO_INSTRUMENTATION = 7;

  // The input app apk does not have a signature.
  NO_SIGNATURE = 20;

  // The test runner class specified by user or in the test APK's manifest file
  // is not compatible with Android Test Orchestrator.
  // Orchestrator is only compatible with AndroidJUnitRunner version 1.1 or
  // higher.
  // Orchestrator can be disabled by using DO_NOT_USE_ORCHESTRATOR
  // OrchestratorOption.
  INSTRUMENTATION_ORCHESTRATOR_INCOMPATIBLE = 18;

  // The test APK does not contain the test runner class specified by the user
  // or in the manifest file. This can be caused by one of the following
  // reasons:
  //
  // - the user provided a runner class name that's incorrect, or
  // - the test runner isn't built into the test APK (might be in the app APK
  // instead).
  NO_TEST_RUNNER_CLASS = 19;

  // A main launcher activity could not be found.
  NO_LAUNCHER_ACTIVITY = 8;

  // The app declares one or more permissions that are not allowed.
  FORBIDDEN_PERMISSIONS = 9;

  // There is a conflict in the provided robo_directives.
  INVALID_ROBO_DIRECTIVES = 10;

  // There is at least one invalid resource name in the provided
  // robo directives
  INVALID_RESOURCE_NAME = 33;

  // Invalid definition of action in the robo directives
  // (e.g. a click or ignore action includes an input text field)
  INVALID_DIRECTIVE_ACTION = 34;

  // There is no test loop intent filter, or the one that is given is
  // not formatted correctly.
  TEST_LOOP_INTENT_FILTER_NOT_FOUND = 12;

  // The request contains a scenario label that was not declared in the
  // manifest.
  SCENARIO_LABEL_NOT_DECLARED = 13;

  // There was an error when parsing a label's value.
  SCENARIO_LABEL_MALFORMED = 14;

  // The request contains a scenario number that was not declared in the
  // manifest.
  SCENARIO_NOT_DECLARED = 15;

  // Device administrator applications are not allowed.
  DEVICE_ADMIN_RECEIVER = 17;

  // The zipped XCTest was malformed. The zip did not contain a single
  // .xctestrun file and the contents of the DerivedData/Build/Products
  // directory.
  MALFORMED_XC_TEST_ZIP = 11;

  // The zipped XCTest was built for the iOS simulator rather than for a
  // physical device.
  BUILT_FOR_IOS_SIMULATOR = 24;

  // The .xctestrun file did not specify any test targets.
  NO_TESTS_IN_XC_TEST_ZIP = 25;

  // One or more of the test targets defined in the .xctestrun file specifies
  // "UseDestinationArtifacts", which is disallowed.
  USE_DESTINATION_ARTIFACTS = 26;

  // XC tests which run on physical devices must have
  // "IsAppHostedTestBundle" == "true" in the xctestrun file.
  TEST_NOT_APP_HOSTED = 28;

  // An Info.plist file in the XCTest zip could not be parsed.
  PLIST_CANNOT_BE_PARSED = 30;

  // The APK is marked as "testOnly".
  // Deprecated and not currently used.
  TEST_ONLY_APK = 21 [deprecated = true];

  // The input IPA could not be parsed.
  MALFORMED_IPA = 22;

  // The application doesn't register the game loop URL scheme.
  MISSING_URL_SCHEME = 35;

  // The iOS application bundle (.app) couldn't be processed.
  MALFORMED_APP_BUNDLE = 36;

  // APK contains no code.
  // See also
  // https://developer.android.com/guide/topics/manifest/application-element.html#code
  NO_CODE_APK = 23;

  // Either the provided input APK path was malformed,
  // the APK file does not exist, or the user does not have permission to
  // access the APK file.
  INVALID_INPUT_APK = 27;

  // APK is built for a preview SDK which is unsupported
  INVALID_APK_PREVIEW_SDK = 29;

  // The matrix expanded to contain too many executions.
  MATRIX_TOO_LARGE = 37;

  // Not enough test quota to run the executions in this matrix.
  TEST_QUOTA_EXCEEDED = 39;

  // A required cloud service api is not activated.
  // See:
  // https://firebase.google.com/docs/test-lab/android/continuous#requirements
  SERVICE_NOT_ACTIVATED = 40;

  // There was an unknown permission issue running this test.
  UNKNOWN_PERMISSION_ERROR = 41;
}

// The state (i.e., progress) of a test execution or matrix.
enum TestState {
  // Do not use.  For proto versioning only.
  TEST_STATE_UNSPECIFIED = 0;

  // The execution or matrix is being validated.
  VALIDATING = 8;

  // The execution or matrix is waiting for resources to become available.
  PENDING = 1;

  // The execution is currently being processed.
  //
  // Can only be set on an execution.
  RUNNING = 2;

  // The execution or matrix has terminated normally.
  //
  // On a matrix this means that the matrix level processing completed normally,
  // but individual executions may be in an ERROR state.
  FINISHED = 3;

  // The execution or matrix has stopped because it encountered an
  // infrastructure failure.
  ERROR = 4;

  // The execution was not run because it corresponds to a unsupported
  // environment.
  //
  // Can only be set on an execution.
  UNSUPPORTED_ENVIRONMENT = 5;

  // The execution was not run because the provided inputs are incompatible with
  // the requested environment.
  //
  // Example: requested AndroidVersion is lower than APK's minSdkVersion
  //
  // Can only be set on an execution.
  INCOMPATIBLE_ENVIRONMENT = 9;

  // The execution was not run because the provided inputs are incompatible with
  // the requested architecture.
  //
  // Example: requested device does not support running the native code in
  // the supplied APK
  //
  // Can only be set on an execution.
  INCOMPATIBLE_ARCHITECTURE = 10;

  // The user cancelled the execution.
  //
  // Can only be set on an execution.
  CANCELLED = 6;

  // The execution or matrix was not run because the provided inputs are not
  // valid.
  //
  // Examples: input file is not of the expected type, is malformed/corrupt, or
  // was flagged as malware
  INVALID = 7;
}

// Outcome summary for a finished test matrix.
enum OutcomeSummary {
  // Do not use. For proto versioning only.
  OUTCOME_SUMMARY_UNSPECIFIED = 0;

  // The test matrix run was successful, for instance:
  //
  // - All the test cases passed.
  // - Robo did not detect a crash of the application under test.
  SUCCESS = 1;

  // A run failed, for instance:
  //
  // - One or more test cases failed.
  // - A test timed out.
  // - The application under test crashed.
  FAILURE = 2;

  // Something unexpected happened. The run should still be considered
  // unsuccessful but this is likely a transient problem and re-running the
  // test might be successful.
  INCONCLUSIVE = 3;

  // All tests were skipped, for instance:
  //
  // - All device configurations were incompatible.
  SKIPPED = 4;
}

// Options for enabling sharding.
message ShardingOption {
  oneof option {
    // Uniformly shards test cases given a total number of shards.
    UniformSharding uniform_sharding = 1;

    // Shards test cases into the specified groups of packages, classes, and/or
    // methods.
    ManualSharding manual_sharding = 2;

    // Shards test based on previous test case timing records.
    SmartSharding smart_sharding = 3;
  }
}

// Uniformly shards test cases given a total number of shards.
//
// For instrumentation tests, it will be translated to "-e numShard" and "-e
// shardIndex" AndroidJUnitRunner arguments. With uniform sharding enabled,
// specifying either of these sharding arguments via `environment_variables` is
// invalid.
//
// Based on the sharding mechanism AndroidJUnitRunner uses, there is no
// guarantee that test cases will be distributed uniformly across all shards.
message UniformSharding {
  // Required. The total number of shards to create. This must always be a
  // positive number that is no greater than the total number of test cases.
  // When you select one or more physical devices, the number of shards must be
  // <= 50. When you select one or more ARM virtual devices, it must be <= 100.
  // When you select only x86 virtual devices, it must be <= 500.
  int32 num_shards = 1;
}

// Shards test cases into the specified groups of packages, classes, and/or
// methods.
//
// With manual sharding enabled, specifying test targets via
// environment_variables or in InstrumentationTest is invalid.
message ManualSharding {
  // Required. Group of packages, classes, and/or test methods to be run for
  // each manually-created shard. You must specify at least one shard if this
  // field is present. When you select one or more physical devices, the number
  // of repeated test_targets_for_shard must be <= 50. When you select one or
  // more ARM virtual devices, it must be <= 100. When you select only x86
  // virtual devices, it must be <= 500.
  repeated TestTargetsForShard test_targets_for_shard = 1;
}

// Test targets for a shard.
message TestTargetsForShard {
  // Group of packages, classes, and/or test methods to be run for each shard.
  // The targets need to be specified in AndroidJUnitRunner argument format. For
  // example, "package com.my.packages" "class com.my.package.MyClass".
  //
  // The number of test_targets must be greater than 0.
  repeated string test_targets = 1;
}

// Shards test based on previous test case timing records.
message SmartSharding {
  // The amount of time tests within a shard should take.
  //
  // Default: 300 seconds (5 minutes).
  // The minimum allowed: 120 seconds (2 minutes).
  //
  // The shard count is dynamically set based on time, up to the maximum shard
  // limit (described below). To guarantee at least one test case for each
  // shard, the number of shards will not exceed the number of test cases. Shard
  // duration will be exceeded if:
  //
  // - The maximum shard limit is reached and there is more calculated test time
  // remaining to allocate into shards.
  // - Any individual test is estimated to be longer than the targeted shard
  // duration.
  //
  // Shard duration is not guaranteed because smart sharding uses test case
  // history and default durations which may not be accurate. The rules for
  // finding the test case timing records are:
  //
  // - If the service has processed a test case in the last 30 days, the record
  //  of the latest successful test case will be used.
  // - For new test cases, the average duration of other known test cases will
  //  be used.
  // - If there are no previous test case timing records available, the default
  //  test case duration is 15 seconds.
  //
  // Because the actual shard duration can exceed the targeted shard duration,
  // we recommend that you set the targeted value at least 5 minutes less than
  // the maximum allowed test timeout (45 minutes for physical devices and 60
  // minutes for virtual), or that you use the custom test timeout value that
  // you set. This approach avoids cancelling the shard before all tests can
  // finish.
  //
  // Note that there is a limit for maximum number of shards. When you select
  // one or more physical devices, the number of shards must be <= 50. When you
  // select one or more ARM virtual devices, it must be <= 100. When you select
  // only x86 virtual devices, it must be <= 500. To guarantee at least one test
  // case for per shard, the number of shards will not exceed the number of test
  // cases. Each shard created counts toward daily test quota.
  google.protobuf.Duration targeted_shard_duration = 1;
}

// Output only. Details about the shard.
message Shard {
  // Output only. The index of the shard among all the shards.
  int32 shard_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The total number of shards.
  int32 num_shards = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Test targets for each shard. Only set for manual sharding.
  TestTargetsForShard test_targets_for_shard = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The estimated shard duration based on previous test case
  // timing records, if available.
  google.protobuf.Duration estimated_shard_duration = 4
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Request to submit a matrix of tests for execution.
message CreateTestMatrixRequest {
  // The GCE project under which this job will run.
  string project_id = 1;

  // The matrix of tests that the user wants to run.
  TestMatrix test_matrix = 2;

  // A string id used to detect duplicated requests.
  // Ids are automatically scoped to a project, so
  // users should ensure the ID is unique per-project.
  // A UUID is recommended.
  //
  // Optional, but strongly recommended.
  string request_id = 3;
}

// Request to get the Test Matrix with the given id.
message GetTestMatrixRequest {
  // Cloud project that owns the test matrix.
  string project_id = 1;

  // Unique test matrix id which was assigned by the service.
  string test_matrix_id = 2;
}

// Request to stop running all of the tests in the specified matrix.
message CancelTestMatrixRequest {
  // Cloud project that owns the test.
  string project_id = 1;

  // Test matrix that will be canceled.
  string test_matrix_id = 2;
}

// Response containing the current state of the specified test matrix.
message CancelTestMatrixResponse {
  // The current rolled-up state of the test matrix.
  // If this state is already final, then the cancelation request will
  // have no effect.
  TestState test_state = 1;
}