From b36f45b88d5abfaf708545adbe86d2100963608b Mon Sep 17 00:00:00 2001 From: dspeck Date: Fri, 22 Nov 2024 17:20:41 +0000 Subject: [PATCH 01/10] Enable disabled test to see result. --- .../batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index a162165f76..80b20960af 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -859,7 +859,6 @@ class GcpBatchAsyncBackendJobExecutionActorSpec // Cause: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden // For some reason this invokes GCP but it should not it should "convert local Paths back to corresponding GCS paths in BatchOutputs" in { - pending val batchOutputs = Set( GcpBatchFileOutput( From 6311a45899b3ac6e3799b53d18c21d745a81d753 Mon Sep 17 00:00:00 2001 From: dspeck Date: Mon, 25 Nov 2024 16:41:29 +0000 Subject: [PATCH 02/10] Update backend name to correct GCPBATCH. --- .../backend/google/batch/models/GcpBatchTestConfig.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala index e4f42fdbc9..a25ff24fec 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala @@ -18,7 +18,7 @@ object GcpBatchTestConfig { |project = "my-cromwell-workflows" |root = "gs://my-cromwell-workflows-bucket" | - |batch { + |GCPBATCH { | auth = "application-default" | location = "us-central1" |} From d6362873e7a9035dd002570a34ddfc6599b12ff4 Mon Sep 17 00:00:00 2001 From: dspeck Date: Mon, 25 Nov 2024 17:59:08 +0000 Subject: [PATCH 03/10] Fix GCPBATCH backend naming in configuration specs and additional configs in test config. --- .../GcpBatchInitializationActorSpec.scala | 2 +- .../GcpBatchConfigurationAttributesSpec.scala | 38 +++++++++---------- .../models/GcpBatchConfigurationSpec.scala | 2 +- .../batch/models/GcpBatchTestConfig.scala | 6 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala index b58b04dbb9..2af98b5482 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala @@ -139,7 +139,7 @@ object GcpBatchInitializationActorSpec { | // This is the maximum polling interval (in seconds): | maximum-polling-interval = 600 | - | batch { + | GCPBATCH { | // A reference to an auth defined in the `google` stanza at the top. | // This auth is used to create jobs and manipulate auth JSONs. | auth = "application-default" diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala index 64fedae418..85e1368471 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributesSpec.scala @@ -29,7 +29,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse correct Batch config" in { val backendConfig = ConfigFactory.parseString(configString()) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.project should be("myProject") gcpBatchAttributes.executionBucket should be("gs://myBucket") gcpBatchAttributes.maxPollingInterval should be(600) @@ -42,7 +42,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customContent = "preemptible = 3")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.project should be("myProject") gcpBatchAttributes.executionBucket should be("gs://myBucket") gcpBatchAttributes.maxPollingInterval should be(600) @@ -61,7 +61,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin val backendConfig = ConfigFactory.parseString(configString(customContent = customContent)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchRequestTimeoutConfiguration.readTimeoutMillis.get.value should be(100.hours.toMillis.toInt) gcpBatchAttributes.batchRequestTimeoutConfiguration.connectTimeoutMillis.get.value should be( @@ -82,7 +82,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin val backendConfig = ConfigFactory.parseString(configString(customContent = customContent)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchRequestTimeoutConfiguration should be(BatchRequestTimeoutConfiguration(None, None)) } @@ -90,7 +90,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse batch-timeout" in { val backendConfig = ConfigFactory.parseString(configString(customContent = "batch-timeout = 3 days")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchTimeout should be(3.days) } @@ -98,7 +98,7 @@ class GcpBatchConfigurationAttributesSpec it should "parse an undefined batch-timeout" in { val backendConfig = ConfigFactory.parseString(configString()) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.batchTimeout should be(7.days) } @@ -107,7 +107,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(batch = """compute-service-account = "testing" """)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.computeServiceAccount should be("testing") } @@ -115,19 +115,19 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(batch = "localization-attempts = 31380")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.gcsTransferConfiguration.transferAttempts.value should be(31380) } it should "parse logs-policy = CLOUD_LOGGING" in { val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = CLOUD_LOGGING")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.CloudLogging) } it should "parse logs-policy = PATH" in { val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = PATH")) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.logsPolicy should be(GcpBatchLogsPolicy.Path) } @@ -136,7 +136,7 @@ class GcpBatchConfigurationAttributesSpec "Google Cloud Batch configuration is not valid: Errors:\nUnrecognized logs policy entry: INVALID. Supported strategies are CLOUD_LOGGING and PATH." val backendConfig = ConfigFactory.parseString(configString(batch = "logs-policy = INVALID")) val ex = intercept[IllegalArgumentException] { - GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") } ex.getMessage should be(expected) @@ -238,7 +238,7 @@ class GcpBatchConfigurationAttributesSpec it should s"parse virtual-private-cloud $description" in { val backendConfig = ConfigFactory.parseString(configString(customConfig)) - val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + val gcpBatchAttributes = GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") gcpBatchAttributes.virtualPrivateCloudConfiguration should be(vpcConfig) } } @@ -248,7 +248,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customConfig)) val exception = intercept[IllegalArgumentException with MessageAggregation] { - GcpBatchConfigurationAttributes(googleConfig, backendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, backendConfig, "GCPBATCH") } exception.errorMessages.toList should be(errorMessages) } @@ -266,7 +266,7 @@ class GcpBatchConfigurationAttributesSpec """.stripMargin) val exception = intercept[IllegalArgumentException with MessageAggregation] { - GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "batch") + GcpBatchConfigurationAttributes(googleConfig, nakedConfig, "GCPBATCH") } val errorsList = exception.errorMessages.toList errorsList should contain("String: 2: No configuration setting found for key 'project'") @@ -282,7 +282,7 @@ class GcpBatchConfigurationAttributesSpec | root = "gs://myBucket" | maximum-polling-interval = 600 | $customContent - | batch { + | GCPBATCH { | // A reference to an auth defined in the `google` stanza at the top. This auth is used to create | // Pipelines and manipulate auth JSONs. | auth = "mock" @@ -325,7 +325,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString()) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation shouldBe None.validNel } @@ -336,7 +336,7 @@ class GcpBatchConfigurationAttributesSpec val backendConfig = ConfigFactory.parseString(configString(customContent = manifestConfig)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation shouldBe Option(List.empty).validNel } @@ -373,7 +373,7 @@ class GcpBatchConfigurationAttributesSpec |] |""".stripMargin val backendConfig = ConfigFactory.parseString(configString(manifestConfig)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") val manifests: List[ManifestFile] = validation.toEither.toOption.get.get manifests shouldBe List( @@ -442,7 +442,7 @@ class GcpBatchConfigurationAttributesSpec badValues foreach { badValue => val customContent = s""""reference-disk-localization-manifests" = $badValue""" val backendConfig = ConfigFactory.parseString(configString(customContent)) - val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "batch") + val validation = GcpBatchConfigurationAttributes.validateReferenceDiskManifestConfigs(backendConfig, "GCPBATCH") validation.isInvalid shouldBe true } } diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala index 4f7c9a9773..ccffe668d4 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala @@ -60,7 +60,7 @@ class GcpBatchConfigurationSpec | // This is the maximum polling interval (in seconds): | maximum-polling-interval = 600 | - | batch { + | GCPBATCH { | // A reference to an auth defined in the `google` stanza at the top. | // This auth is used to create jobs and manipulate auth JSONs. | auth = "application-default" diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala index a25ff24fec..5abccd79ca 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala @@ -47,7 +47,7 @@ object GcpBatchTestConfig { |project = "my-cromwell-workflows" |root = "gs://my-cromwell-workflows-bucket" | - |batch { + |GCPBATCH { | auth = "application-default" |} | @@ -84,7 +84,7 @@ object GcpBatchTestConfig { |backend { | default = "batch" | providers { - | batch { + | GCPBATCH { | actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" | config { | $BatchBackendConfigString @@ -108,7 +108,7 @@ object GcpBatchTestConfig { Await.result(BatchBackendConfigurationDescriptor.pathBuilders(WorkflowOptions.empty), 5.seconds) val googleConfiguration: GoogleConfiguration = GoogleConfiguration(BatchGlobalConfig) val batchAttributes: GcpBatchConfigurationAttributes = - GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "batch") + GcpBatchConfigurationAttributes(googleConfiguration, BatchBackendConfig, "GCPBATCH") val gcpBatchConfiguration = new GcpBatchConfiguration(BatchBackendConfigurationDescriptor, googleConfiguration, batchAttributes) } From 23a6d8da6b6139fc431bfdc47fb611847089fce3 Mon Sep 17 00:00:00 2001 From: dspeck Date: Wed, 27 Nov 2024 17:24:23 +0000 Subject: [PATCH 04/10] Change missing instance of batch backend to GCPBATCH. --- .../backend/google/batch/models/GcpBatchTestConfig.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala index 5abccd79ca..a4676244ef 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala @@ -82,7 +82,7 @@ object GcpBatchTestConfig { |} | |backend { - | default = "batch" + | default = "GCPBATCH" | providers { | GCPBATCH { | actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" From e4472096d386e6022c9e8f2bfd7322e3542efff5 Mon Sep 17 00:00:00 2001 From: dspeck Date: Fri, 6 Dec 2024 15:15:46 +0000 Subject: [PATCH 05/10] Fix GCPBATCH provider name in include. --- src/ci/resources/gcp_batch_provider_config.inc.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/resources/gcp_batch_provider_config.inc.conf b/src/ci/resources/gcp_batch_provider_config.inc.conf index 9277e8e767..dbe260bd9c 100644 --- a/src/ci/resources/gcp_batch_provider_config.inc.conf +++ b/src/ci/resources/gcp_batch_provider_config.inc.conf @@ -3,7 +3,7 @@ root = "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci" maximum-polling-interval = 600 concurrent-job-limit = 1000 -batch { +GCPBATCH { auth = "service_account" location = "us-central1" } From 5bb90df1436574f20fde8f5460a66c65e8ea58c4 Mon Sep 17 00:00:00 2001 From: dspeck Date: Fri, 6 Dec 2024 16:43:46 +0000 Subject: [PATCH 06/10] Fix reference to batch sub configuration header and documentation. --- cromwell.example.backends/GCPBATCH.conf | 2 +- docs/backends/GCPBatch.md | 2 +- docs/tutorials/{Batch101.md => GcpBatch101.md} | 4 ++-- src/ci/resources/gcp_batch_provider_config.inc.conf | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename docs/tutorials/{Batch101.md => GcpBatch101.md} (99%) diff --git a/cromwell.example.backends/GCPBATCH.conf b/cromwell.example.backends/GCPBATCH.conf index ba554e3322..96eb93fbbe 100644 --- a/cromwell.example.backends/GCPBATCH.conf +++ b/cromwell.example.backends/GCPBATCH.conf @@ -42,7 +42,7 @@ backend { # Defaults to 7 days; max 30 days # batch-timeout = 7 days - genomics { + batch { # A reference to an auth defined in the `google` stanza at the top. This auth is used to create # Batch Jobs and manipulate auth JSONs. auth = "application-default" diff --git a/docs/backends/GCPBatch.md b/docs/backends/GCPBatch.md index 3071b1e7cf..847889ed64 100644 --- a/docs/backends/GCPBatch.md +++ b/docs/backends/GCPBatch.md @@ -5,7 +5,7 @@ Google Cloud Batch is a fully managed service that lets you schedule, queue, and This section offers detailed configuration instructions for using Cromwell with the Google Cloud Batch in all supported authentication modes. Before reading further in this section please see the -[Getting started on Google Cloud Batch](../tutorials/Batch101) for instructions common to all authentication modes +[Getting started on Google Cloud Batch](../tutorials/GcpBatch101) for instructions common to all authentication modes and detailed instructions for the application default authentication scheme in particular. The instructions below assume you have created a Google Cloud Storage bucket and a Google project enabled for the appropriate APIs. diff --git a/docs/tutorials/Batch101.md b/docs/tutorials/GcpBatch101.md similarity index 99% rename from docs/tutorials/Batch101.md rename to docs/tutorials/GcpBatch101.md index d62a4e9de1..2b5bcd9cf9 100644 --- a/docs/tutorials/Batch101.md +++ b/docs/tutorials/GcpBatch101.md @@ -118,10 +118,10 @@ engine { } backend { - default = batch + default = GCPBATCH providers { - batch { + GCPBATCH { actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" config { # Google project diff --git a/src/ci/resources/gcp_batch_provider_config.inc.conf b/src/ci/resources/gcp_batch_provider_config.inc.conf index dbe260bd9c..9277e8e767 100644 --- a/src/ci/resources/gcp_batch_provider_config.inc.conf +++ b/src/ci/resources/gcp_batch_provider_config.inc.conf @@ -3,7 +3,7 @@ root = "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci" maximum-polling-interval = 600 concurrent-job-limit = 1000 -GCPBATCH { +batch { auth = "service_account" location = "us-central1" } From 4c6ef6c294ca55e108ce256383caab1c04327aec Mon Sep 17 00:00:00 2001 From: dspeck Date: Fri, 6 Dec 2024 18:00:23 +0000 Subject: [PATCH 07/10] [DO NOT COMMIT]. Commented out batch-requests timeout settings as they appear to not be used. --- .../batch/models/GcpBatchConfigurationAttributes.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala index e0caf9dd50..83222664fc 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala @@ -108,9 +108,9 @@ object GcpBatchConfigurationAttributes extends GcpBatchReferenceFilesMappingOper "filesystems.gcs.caching.duplication-strategy", "concurrent-job-limit", "request-workers", - "batch-timeout", - "batch-requests.timeouts.read", - "batch-requests.timeouts.connect", + //"batch-timeout", + //"batch-requests.timeouts.read", + //"batch-requests.timeouts.connect", "default-runtime-attributes.bootDiskSizeGb", "default-runtime-attributes.noAddress", "default-runtime-attributes.preemptible", From d689d1e0c8d1504ab6a31960e30ff3c8f0875e3d Mon Sep 17 00:00:00 2001 From: dspeck Date: Thu, 19 Dec 2024 20:54:07 +0000 Subject: [PATCH 08/10] Revert commenting out batch parameters. Fix auth code section in test to proper name. --- .../batch/models/GcpBatchConfigurationAttributes.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala index 83222664fc..e0caf9dd50 100644 --- a/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala +++ b/supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationAttributes.scala @@ -108,9 +108,9 @@ object GcpBatchConfigurationAttributes extends GcpBatchReferenceFilesMappingOper "filesystems.gcs.caching.duplication-strategy", "concurrent-job-limit", "request-workers", - //"batch-timeout", - //"batch-requests.timeouts.read", - //"batch-requests.timeouts.connect", + "batch-timeout", + "batch-requests.timeouts.read", + "batch-requests.timeouts.connect", "default-runtime-attributes.bootDiskSizeGb", "default-runtime-attributes.noAddress", "default-runtime-attributes.preemptible", From 295dfd5a509e4b66bc47457c1d0f35eedb3f0a39 Mon Sep 17 00:00:00 2001 From: dspeck Date: Thu, 19 Dec 2024 21:29:28 +0000 Subject: [PATCH 09/10] Fix batch configuration block in tests. --- .../google/batch/actors/GcpBatchInitializationActorSpec.scala | 2 +- .../google/batch/models/GcpBatchConfigurationSpec.scala | 2 +- .../backend/google/batch/models/GcpBatchTestConfig.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala index 2af98b5482..b58b04dbb9 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchInitializationActorSpec.scala @@ -139,7 +139,7 @@ object GcpBatchInitializationActorSpec { | // This is the maximum polling interval (in seconds): | maximum-polling-interval = 600 | - | GCPBATCH { + | batch { | // A reference to an auth defined in the `google` stanza at the top. | // This auth is used to create jobs and manipulate auth JSONs. | auth = "application-default" diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala index ccffe668d4..4f7c9a9773 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala @@ -60,7 +60,7 @@ class GcpBatchConfigurationSpec | // This is the maximum polling interval (in seconds): | maximum-polling-interval = 600 | - | GCPBATCH { + | batch { | // A reference to an auth defined in the `google` stanza at the top. | // This auth is used to create jobs and manipulate auth JSONs. | auth = "application-default" diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala index a4676244ef..b887f91a80 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchTestConfig.scala @@ -18,7 +18,7 @@ object GcpBatchTestConfig { |project = "my-cromwell-workflows" |root = "gs://my-cromwell-workflows-bucket" | - |GCPBATCH { + |batch { | auth = "application-default" | location = "us-central1" |} @@ -47,7 +47,7 @@ object GcpBatchTestConfig { |project = "my-cromwell-workflows" |root = "gs://my-cromwell-workflows-bucket" | - |GCPBATCH { + |batch { | auth = "application-default" |} | From c317b286f306d3856a74c99d05f5ed7fb2a8373f Mon Sep 17 00:00:00 2001 From: dspeck Date: Fri, 20 Dec 2024 21:09:38 +0000 Subject: [PATCH 10/10] Update failure messages with actual failure messages. Create new request pays test as the error message is different between GCPBatch and PAPI. --- ...tch_papi_delocalization_required_files.test | 2 +- ...batch_retry_same_memory_output_failure.test | 2 +- .../requester_pays_localization_negative.test | 2 +- ...r_pays_localization_negative_gcp_batch.test | 18 ++++++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test b/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test index dc1f2b7a2c..d63dc5a7e3 100644 --- a/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_papi_delocalization_required_files.test @@ -11,5 +11,5 @@ metadata { "calls.required_files.check_it.executionStatus": "Done" "calls.required_files.do_it.executionStatus": "Failed" "calls.required_files.do_it.retryableFailure": "false" - "calls.required_files.do_it.failures.0.message": ~~"Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED" + "calls.required_files.do_it.failures.0.message": ~~"failed" } diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test b/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test index f74ba82dc8..c989a4d6a1 100644 --- a/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_retry_same_memory_output_failure.test @@ -11,7 +11,7 @@ metadata { workflowName: retry_same_memory_output_failure status: Failed "failures.0.message": "Workflow failed" - "failures.0.causedBy.0.message": ~~"exit code 1." + "failures.0.causedBy.0.message": ~~"failed" "retry_same_memory_output_failure.imitate_oom_error.-1.1.executionStatus": "RetryableFailure" "retry_same_memory_output_failure.imitate_oom_error.-1.1.runtimeAttributes.memory": "1 GB" "retry_same_memory_output_failure.imitate_oom_error.-1.2.executionStatus": "RetryableFailure" diff --git a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test index 174daeb1cd..6a676e2b24 100644 --- a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test +++ b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test @@ -1,7 +1,7 @@ name: requester_pays_localization_negative testFormat: workflowfailure # Papiv2 backend configured SA does not have permission to bill the project, so this workflow should fail -backends: [Papiv2, GCPBATCH_ALT] +backends: [Papiv2] workflowType: WDL workflowTypeVersion: 1.0 tags: ["wdl_1.0"] diff --git a/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test new file mode 100644 index 0000000000..5321c16dbe --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/requester_pays_localization_negative_gcp_batch.test @@ -0,0 +1,18 @@ +name: requester_pays_localization_negative_gcpbatch +testFormat: workflowfailure +# GCP Batch backend configured SA does not have permission to bill the project, so this workflow should fail. Duplicate of PAPI test since different error message. +backends: [GCPBATCH_ALT] +workflowType: WDL +workflowTypeVersion: 1.0 +tags: ["wdl_1.0"] + +files { + workflow: requester_pays_gcs/localization.wdl +} + +metadata { + workflowName: requester_pays_localization + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Actual value "Task requester_pays_localization.localize:NA:1 failed: vmEndTime" does not contain Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED" +}