diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test b/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test index dc1f2b7a2ce..d63dc5a7e35 100644 --- a/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test @@ -11,5 +11,5 @@ metadata { "calls.required_files.check_it.executionStatus": "Done" "calls.required_files.do_it.executionStatus": "Failed" "calls.required_files.do_it.retryableFailure": "false" - "calls.required_files.do_it.failures.0.message": ~~"Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED" + "calls.required_files.do_it.failures.0.message": ~~"failed" } diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test b/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test index f74ba82dc84..c989a4d6a1d 100644 --- a/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test @@ -11,7 +11,7 @@ metadata { workflowName: retry_same_memory_output_failure status: Failed "failures.0.message": "Workflow failed" - "failures.0.causedBy.0.message": ~~"exit code 1." + "failures.0.causedBy.0.message": ~~"failed" "retry_same_memory_output_failure.imitate_oom_error.-1.1.executionStatus": "RetryableFailure" "retry_same_memory_output_failure.imitate_oom_error.-1.1.runtimeAttributes.memory": "1 GB" "retry_same_memory_output_failure.imitate_oom_error.-1.2.executionStatus": "RetryableFailure" diff --git a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test index 174daeb1cd5..6a676e2b241 100644 --- a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test +++ b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test @@ -1,7 +1,7 @@ name: requester_pays_localization_negative testFormat: workflowfailure # Papiv2 backend configured SA does not have permission to bill the project, so this workflow should fail -backends: [Papiv2, GCPBATCH_ALT] +backends: [Papiv2] workflowType: WDL workflowTypeVersion: 1.0 tags: ["wdl_1.0"] diff --git a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test new file mode 100644 index 00000000000..5321c16dbe3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test @@ -0,0 +1,18 @@ +name: requester_pays_localization_negative_gcpbatch +testFormat: workflowfailure +# GCP Batch backend configured SA does not have permission to bill the project, so this workflow should fail. Duplicate of PAPI test since different error message. +backends: [GCPBATCH_ALT] +workflowType: WDL +workflowTypeVersion: 1.0 +tags: ["wdl_1.0"] + +files { + workflow: requester_pays_gcs/localization.wdl +} + +metadata { + workflowName: requester_pays_localization + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Actual value "Task requester_pays_localization.localize:NA:1 failed: vmEndTime" does not contain Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED" +} diff --git a/cromwell.example.backends/GCPBATCH.conf b/cromwell.example.backends/GCPBATCH.conf index ba554e3322d..96eb93fbbe5 100644 --- a/cromwell.example.backends/GCPBATCH.conf +++ b/cromwell.example.backends/GCPBATCH.conf @@ -42,7 +42,7 @@ backend { # Defaults to 7 days; max 30 days # batch-timeout = 7 days - genomics { + batch { # A reference to an auth defined in the `google` stanza at the top. This auth is used to create # Batch Jobs and manipulate auth JSONs. auth = "application-default" diff --git a/docs/backends/GCPBatch.md b/docs/backends/GCPBatch.md index 3071b1e7cfa..847889ed649 100644 --- a/docs/backends/GCPBatch.md +++ b/docs/backends/GCPBatch.md @@ -5,7 +5,7 @@ Google Cloud Batch is a fully managed service that lets you schedule, queue, and This section offers detailed configuration instructions for using Cromwell with the Google Cloud Batch in all supported authentication modes. Before reading further in this section please see the -[Getting started on Google Cloud Batch](../tutorials/Batch101) for instructions common to all authentication modes +[Getting started on Google Cloud Batch](../tutorials/GcpBatch101) for instructions common to all authentication modes and detailed instructions for the application default authentication scheme in particular. The instructions below assume you have created a Google Cloud Storage bucket and a Google project enabled for the appropriate APIs. diff --git a/docs/tutorials/Batch101.md b/docs/tutorials/GcpBatch101.md similarity index 99% rename from docs/tutorials/Batch101.md rename to docs/tutorials/GcpBatch101.md index d62a4e9de1a..2b5bcd9cf99 100644 --- a/docs/tutorials/Batch101.md +++ b/docs/tutorials/GcpBatch101.md @@ -118,10 +118,10 @@ engine { } backend { - default = batch + default = GCPBATCH providers { - batch { + GCPBATCH { actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" config { # Google project diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index a162165f765..80b20960af6 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -859,7 +859,6 @@ class GcpBatchAsyncBackendJobExecutionActorSpec // Cause: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden // For some reason this invokes GCP but it should not it should "convert local Paths back to corresponding GCS paths in BatchOutputs" in { - pending val batchOutputs = Set( GcpBatchFileOutput( diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala index 64fedae4186..85e1368471a 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala @@ -29,7 +29,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse correct Batch config" in { val backendConfig = ConfigFactory.parseString(configString()) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.project should be("myProject") gcpBatchAttributes.executionBucket should be("gs://myBucket") gcpBatchAttributes.maxPollingInterval should be(600) @@ -42,7 +42,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customContent = "preemptible = 3")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.project should be("myProject") gcpBatchAttributes.executionBucket should be("gs://myBucket") gcpBatchAttributes.maxPollingInterval should be(600) @@ -61,7 +61,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin val backendConfig = ConfigFactory.parseString(configString(customContent = customContent)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchRequestTimeoutConfiguration.readTimeoutMillis.get.value should be(100.hours.toMillis.toInt) gcpBatchAttributes.batchRequestTimeoutConfiguration.connectTimeoutMillis.get.value should be( @@ -82,7 +82,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin val backendConfig = ConfigFactory.parseString(configString(customContent = customContent)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchRequestTimeoutConfiguration should be(BatchRequestTimeoutConfiguration(None, None)) } @@ -90,7 +90,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse batch-timeout" in { val backendConfig = ConfigFactory.parseString(configString(customContent = "batch-timeout = 3 days")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchTimeout should be(3.days) } @@ -98,7 +98,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse an undefined batch-timeout" in { val backendConfig = ConfigFactory.parseString(configString()) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchTimeout should be(7.days) } @@ -107,7 +107,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(batch = """compute-service-account = "testing" """)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.computeServiceAccount should be("testing") } @@ -115,19 +115,19 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(batch = "localization-attempts = 31380")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.gcsTransferConfiguration.transferAttempts.value should be(31380) } it should "parse logs-policy = CLOUD_LOGGING" in { val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = CLOUD_LOGGING")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.CloudLogging) } it should "parse logs-policy = PATH" in { val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = PATH")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.Path) } @@ -136,7 +136,7 @@ class GcpBatchConfigurationAttributesSpec "Google Cloud Batch configuration is not valid: Errors:\nUnrecognized logs policy entry: INVALID. Supported strategies are CLOUD_LOGGING and PATH." val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = INVALID")) val ex = intercept[IllegalArgumentException] { - GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") } ex.getMessage should be(expected) @@ -238,7 +238,7 @@ class GcpBatchConfigurationAttributesSpec it should s"parse virtual-private-cloud $description" in { val backendConfig = ConfigFactory.parseString(configString(customConfig)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.virtualPrivateCloudConfiguration should be(vpcConfig) } } @@ -248,7 +248,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customConfig)) val exception = intercept[IllegalArgumentException with MessageAggregation] { - GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") } exception.errorMessages.toList should be(errorMessages) } @@ -266,7 +266,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin) val exception = intercept[IllegalArgumentException with MessageAggregation] { - GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "GCPBATCH") } val errorsList = exception.errorMessages.toList errorsList should contain("String: 2: No configuration setting found for key 'project'") @@ -282,7 +282,7 @@ class GcpBatchConfigurationAttributesSpec | root = "gs://myBucket" | maximum-polling-interval = 600 | $customContent - | batch { + | GCPBATCH { | // A reference to an auth defined in the `google` stanza at the top. This auth is used to create | // Pipelines and manipulate auth JSONs. | auth = "mock" @@ -325,7 +325,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString()) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation shouldBe None.validNel } @@ -336,7 +336,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customContent = manifestConfig)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation shouldBe Option(List.empty).validNel } @@ -373,7 +373,7 @@ class GcpBatchConfigurationAttributesSpec |] |""".stripMargin val backendConfig = ConfigFactory.parseString(configString(manifestConfig)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") val manifests: List[ManifestFile] = validation.toEither.toOption.get.get manifests shouldBe List( @@ -442,7 +442,7 @@ class GcpBatchConfigurationAttributesSpec badValues foreach { badValue => val customContent = s""""reference-disk-localization-manifests" = $badValue""" val backendConfig = ConfigFactory.parseString(configString(customContent)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation.isInvalid shouldBe true } } diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala index e4f42fdbc94..b887f91a80e 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala @@ -82,9 +82,9 @@ object GcpBatchTestConfig { |} | |backend { - | default = "batch" + | default = "GCPBATCH" | providers { - | batch { + | GCPBATCH { | actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" | config { | $BatchBackendConfigString @@ -108,7 +108,7 @@ object GcpBatchTestConfig { Await.result(BatchBackendConfigurationDescriptor.pathBuilders(WorkflowOptions.empty), 5.seconds) val googleConfiguration: GoogleConfiguration = GoogleConfiguration(BatchGlobalConfig) val batchAttributes: GcpBatchConfigurationAttributes = - GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "GCPBATCH") val gcpBatchConfiguration = new GcpBatchConfiguration(BatchBackendConfigurationDescriptor, googleConfiguration, batchAttributes) }