From da6ac2b8e687056da7fb33c5ea11dc7b5f2063ba Mon Sep 17 00:00:00 2001 From: kevinRenaersBio Date: Mon, 19 Jun 2023 12:56:06 +0000 Subject: [PATCH 1/2] Support EFS volumes for AWS. --- .../backend/impl/aws/io/AwsBatchVolume.scala | 15 +++++++++++++-- .../impl/aws/AwsBatchRuntimeAttributesSpec.scala | 6 ++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala index 769c2c2f5d8..83544560846 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala @@ -32,7 +32,7 @@ package cromwell.backend.impl.aws.io import cats.data.Validated._ import cats.syntax.validated._ -import software.amazon.awssdk.services.batch.model.{Host, MountPoint, Volume} +import software.amazon.awssdk.services.batch.model.{Host, MountPoint, Volume, EFSVolumeConfiguration} import cromwell.core.path.{DefaultPathBuilder, Path} import common.exception.MessageAggregation import common.validation.ErrorOr._ @@ -54,6 +54,7 @@ object AwsBatchVolume { // In AWS, disks are auto-sized so these patterns match simply "local-disk" or "/some/mnt" val MountedDiskPattern: Regex = raw"""^\s*(${DiskPatterns.Directory})\s*$$""".r val LocalDiskPattern: Regex = raw"""^\s*local-disk\s*$$""".r + val EFSPattern: Regex = raw"""^efs:([^\s]+)\s+(\/[^\s]+)\s+fs-([^\s]*)\s+(ENABLED|DISABLED)""".r def parse(s: String): Try[AwsBatchVolume] = { @@ -62,13 +63,15 @@ object AwsBatchVolume { Valid(AwsBatchWorkingDisk()) case MountedDiskPattern(mountPoint) => Valid(AwsBatchEmptyMountedDisk(DefaultPathBuilder.get(mountPoint))) + case EFSPattern(name, mountPoint, fileSystemId, encryption) => + Valid(AwsBatchEFSDisk(name, DefaultPathBuilder.get(mountPoint), fileSystemId, encryption)) // In addition to the AWS-specific patterns above, we can also fall back to PAPI-style patterns and ignore the size case DiskPatterns.WorkingDiskPattern(_, _) => Valid(AwsBatchWorkingDisk()) case DiskPatterns.MountedDiskPattern(mountPoint, _, fsType) => Valid(AwsBatchEmptyMountedDisk(DefaultPathBuilder.get(mountPoint),fsType)) case _ => - s"Disk strings should be of the format 'local-disk' or '/mount/point' but got: '$s'".invalidNel + s"Disk strings should be of the format 'local-disk' or '/mount/point' or 'efs:name /mount/point fs-id ENABLED' but got: '$s'".invalidNel } Try(validation match { @@ -86,6 +89,7 @@ trait AwsBatchVolume { def name: String def mountPoint: Path def fsType: String + def efsVolumeConfiguration: EFSVolumeConfiguration=null def getHostPath(id: Option[String]) : String = { id match { case Some(id) => mountPoint.toAbsolutePath.pathAsString + "/" + id @@ -97,6 +101,7 @@ trait AwsBatchVolume { .builder .name(name) .host(Host.builder.sourcePath(getHostPath(id)).build) + .efsVolumeConfiguration(efsVolumeConfiguration) .build } def toMountPoint: MountPoint = { @@ -127,3 +132,9 @@ case class AwsBatchWorkingDisk() extends AwsBatchVolume { val fsType = AwsBatchWorkingDisk.fsType override def toString: String = s"$name $mountPoint" } + +case class AwsBatchEFSDisk(name: String, mountPoint: Path, fileSystemId: String, encryption: String) extends AwsBatchVolume { + val fsType = "efs" + override val efsVolumeConfiguration = EFSVolumeConfiguration.builder.fileSystemId(fileSystemId).transitEncryption(encryption).build() + override def toString: String = s"$name $mountPoint efs-volume" +} diff --git a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala index fbab5a81a23..c5ab300403f 100644 --- a/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala @@ -221,6 +221,12 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } + "validate a valid efs disks entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "disks" -> WomString("efs:data /data fs-123abc ENABLED")) + val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(AwsBatchVolume.parse("efs:data /data fs-123abc ENABLED").get)) + assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + "fail to validate an invalid disks entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "disks" -> WomInteger(10)) assertAwsBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disks runtime attribute to be a comma separated String or Array[String]") From c8d99485a9473e26edcdc79db62aecfd116761d9 Mon Sep 17 00:00:00 2001 From: kevinRenaersBio Date: Mon, 19 Jun 2023 13:31:35 +0000 Subject: [PATCH 2/2] Trigger action. --- .../scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala index 83544560846..ecc40496191 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala @@ -135,6 +135,10 @@ case class AwsBatchWorkingDisk() extends AwsBatchVolume { case class AwsBatchEFSDisk(name: String, mountPoint: Path, fileSystemId: String, encryption: String) extends AwsBatchVolume { val fsType = "efs" - override val efsVolumeConfiguration = EFSVolumeConfiguration.builder.fileSystemId(fileSystemId).transitEncryption(encryption).build() + override val efsVolumeConfiguration = EFSVolumeConfiguration + .builder + .fileSystemId(fileSystemId) + .transitEncryption(encryption) + .build() override def toString: String = s"$name $mountPoint efs-volume" }