Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support EFS volumes for AWS. #7162

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

Support EFS volumes for AWS. #7162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da6ac2b
Support EFS volumes for AWS.
kevinRenaersBio Jun 19, 2023
c8d9948
Trigger action.
kevinRenaersBio Jun 19, 2023
7790968
Merge branch 'develop' into aws_efs_volume
kevinRenaersBio Jun 22, 2023
3048155
Merge branch 'develop' into aws_efs_volume
kevinRenaersBio Jun 30, 2023
f0687d7
Merge branch 'develop' into aws_efs_volume
kevinRenaersBio Jul 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchVolume.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ package cromwell.backend.impl.aws.io

import cats.data.Validated._
import cats.syntax.validated._
import software.amazon.awssdk.services.batch.model.{Host, MountPoint, Volume}
import software.amazon.awssdk.services.batch.model.{Host, MountPoint, Volume, EFSVolumeConfiguration}
import cromwell.core.path.{DefaultPathBuilder, Path}
import common.exception.MessageAggregation
import common.validation.ErrorOr._
Expand All @@ -54,6 +54,7 @@ object AwsBatchVolume {
// In AWS, disks are auto-sized so these patterns match simply "local-disk" or "/some/mnt"
val MountedDiskPattern: Regex = raw"""^\s*(${DiskPatterns.Directory})\s*$$""".r
val LocalDiskPattern: Regex = raw"""^\s*local-disk\s*$$""".r
val EFSPattern: Regex = raw"""^efs:([^\s]+)\s+(\/[^\s]+)\s+fs-([^\s]*)\s+(ENABLED|DISABLED)""".r

def parse(s: String): Try[AwsBatchVolume] = {

Expand All @@ -62,13 +63,15 @@ object AwsBatchVolume {
Valid(AwsBatchWorkingDisk())
case MountedDiskPattern(mountPoint) =>
Valid(AwsBatchEmptyMountedDisk(DefaultPathBuilder.get(mountPoint)))
case EFSPattern(name, mountPoint, fileSystemId, encryption) =>
Valid(AwsBatchEFSDisk(name, DefaultPathBuilder.get(mountPoint), fileSystemId, encryption))
// In addition to the AWS-specific patterns above, we can also fall back to PAPI-style patterns and ignore the size
case DiskPatterns.WorkingDiskPattern(_, _) =>
Valid(AwsBatchWorkingDisk())
case DiskPatterns.MountedDiskPattern(mountPoint, _, fsType) =>
Valid(AwsBatchEmptyMountedDisk(DefaultPathBuilder.get(mountPoint),fsType))
case _ =>
s"Disk strings should be of the format 'local-disk' or '/mount/point' but got: '$s'".invalidNel
s"Disk strings should be of the format 'local-disk' or '/mount/point' or 'efs:name /mount/point fs-id ENABLED' but got: '$s'".invalidNel
}

Try(validation match {
Expand All @@ -86,6 +89,7 @@ trait AwsBatchVolume {
def name: String
def mountPoint: Path
def fsType: String
def efsVolumeConfiguration: EFSVolumeConfiguration=null
def getHostPath(id: Option[String]) : String = {
id match {
case Some(id) => mountPoint.toAbsolutePath.pathAsString + "/" + id
Expand All @@ -97,6 +101,7 @@ trait AwsBatchVolume {
.builder
.name(name)
.host(Host.builder.sourcePath(getHostPath(id)).build)
.efsVolumeConfiguration(efsVolumeConfiguration)
.build
}
def toMountPoint: MountPoint = {
Expand Down Expand Up @@ -127,3 +132,13 @@ case class AwsBatchWorkingDisk() extends AwsBatchVolume {
val fsType = AwsBatchWorkingDisk.fsType
override def toString: String = s"$name $mountPoint"
}

case class AwsBatchEFSDisk(name: String, mountPoint: Path, fileSystemId: String, encryption: String) extends AwsBatchVolume {
val fsType = "efs"
override val efsVolumeConfiguration = EFSVolumeConfiguration
.builder
.fileSystemId(fileSystemId)
.transitEncryption(encryption)
.build()
override def toString: String = s"$name $mountPoint efs-volume"
}
6 changes: 6 additions & 0 deletions ...Backends/aws/src/test/scala/cromwell/backend/impl/aws/AwsBatchRuntimeAttributesSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,12 @@ class AwsBatchRuntimeAttributesSpec extends AnyWordSpecLike with CromwellTimeout
assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
}

"validate a valid efs disks entry" in {
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "disks" -> WomString("efs:data /data fs-123abc ENABLED"))
val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(AwsBatchVolume.parse("efs:data /data fs-123abc ENABLED").get))
assertAwsBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
}

"fail to validate an invalid disks entry" in {
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "scriptBucketName" -> WomString("my-stuff"), "disks" -> WomInteger(10))
assertAwsBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disks runtime attribute to be a comma separated String or Array[String]")
Expand Down