diff --git a/.gitignore b/.gitignore index 8bb22129f381..26fc540918a1 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ doc/source/tutorial/services.rst # Pyenv .python-version +.env diff --git a/awscli/customizations/cloudformation/artifact_exporter.py b/awscli/customizations/cloudformation/artifact_exporter.py index 7fba8dddbba3..e9137efbf31d 100644 --- a/awscli/customizations/cloudformation/artifact_exporter.py +++ b/awscli/customizations/cloudformation/artifact_exporter.py @@ -25,11 +25,14 @@ from awscli.customizations.cloudformation import exceptions from awscli.customizations.cloudformation.yamlhelper import yaml_dump, \ yaml_parse +from awscli.customizations.cloudformation import modules import jmespath LOG = logging.getLogger(__name__) +MODULES = "Modules" +RESOURCES = "Resources" def is_path_value_valid(path): return isinstance(path, str) @@ -591,8 +594,8 @@ def __init__(self, template_path, parent_dir, uploader, raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}" .format(parent_dir)) - abs_template_path = make_abs_path(parent_dir, template_path) + self.module_parent_path = abs_template_path template_dir = os.path.dirname(abs_template_path) with open(abs_template_path, "r") as handle: @@ -651,14 +654,39 @@ def export(self): :return: The template with references to artifacts that have been exported to s3. """ + + # Process modules + try: + self.template_dict = modules.process_module_section( + self.template_dict, + self.template_dir, + self.module_parent_path) + except Exception as e: + msg=f"Failed to process Modules section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + self.template_dict = self.export_metadata(self.template_dict) - if "Resources" not in self.template_dict: + if RESOURCES not in self.template_dict: return self.template_dict + # Process modules that are specified as Resources, not in Modules + try: + self.template_dict = modules.process_resources_section( + self.template_dict, + self.template_dir, + self.module_parent_path, + None) + except Exception as e: + msg=f"Failed to process modules in Resources: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.template_dict = self.export_global_artifacts(self.template_dict) - self.export_resources(self.template_dict["Resources"]) + self.export_resources(self.template_dict[RESOURCES]) return self.template_dict diff --git a/awscli/customizations/cloudformation/exceptions.py b/awscli/customizations/cloudformation/exceptions.py index b2625cdd27f9..4beec17f6503 100644 --- a/awscli/customizations/cloudformation/exceptions.py +++ b/awscli/customizations/cloudformation/exceptions.py @@ -57,3 +57,9 @@ class DeployBucketRequiredError(CloudFormationCommandError): class InvalidForEachIntrinsicFunctionError(CloudFormationCommandError): fmt = 'The value of {resource_id} has an invalid "Fn::ForEach::" format: Must be a list of three entries' + +class InvalidModulePathError(CloudFormationCommandError): + fmt = 'The value of {source} is not a valid path to a local file' + +class InvalidModuleError(CloudFormationCommandError): + fmt = 'Invalid module: {msg}' diff --git a/awscli/customizations/cloudformation/modules.py b/awscli/customizations/cloudformation/modules.py new file mode 100644 index 000000000000..fceb5780c870 --- /dev/null +++ b/awscli/customizations/cloudformation/modules.py @@ -0,0 +1,789 @@ +# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +This file implements local module support for the package command + +See tests/unit/customizations/cloudformation/modules for examples of what the +Modules section of a template looks like. + +Modules can be referenced in a new Modules section in the template, or they can +be referenced as Resources with the Type LocalModule. Modules have a Source +attribute pointing to a local file, a Properties attribute that corresponds to +Parameters in the modules, and an Overrides attribute that can override module +output. + +The `Modules` section. + +```yaml +Modules: + Content: + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +``` + +A module configured as a `Resource`. + +```yaml +Resources: + Content: + Type: LocalModule + Source: ./module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +``` + +A module is itself basically a CloudFormation template, with a Parameters +section and Resources that are injected into the parent template. The +Properties defined in the Modules section correspond to the Parameters in the +module. These modules operate in a similar way to registry modules. + +The name of the module in the Modules section is used as a prefix to logical +ids that are defined in the module. Or if the module is referenced in the Type +attribute of a Resource, the logical id of the resource is used as the prefix. + +In addition to the parent setting Properties, all attributes of the module can +be overridden with Overrides, which require the consumer to know how the module +is structured. This "escape hatch" is considered a first class citizen in the +design, to avoid excessive Parameter definitions to cover every possible use +case. One caveat is that using Overrides is less stable, since the module +author might change logical ids. Using module Outputs can mitigate this. + +Module Parameters (set by Properties in the parent) are handled with Refs, +Subs, and GetAtts in the module. These are handled in a way that fixes +references to match module prefixes, fully resolving values that are actually +strings and leaving others to be resolved at deploy time. + +Modules can contain other modules, with no enforced limit to the levels of nesting. + +Modules can define Outputs, which are key-value pairs that can be referenced by +the parent. + +When using modules, you can use a comma-delimited list to create a number of +similar resources. This is simpler than using `Fn::ForEach` but has the +limitation of requiring the list to be resolved at build time. +See tests/unit/customizations/cloudformation/modules/vpc-module.yaml. + +An example of a Map is defining subnets in a VPC. + +```yaml +Parameters: + CidrBlock: + Type: String + PrivateCidrBlocks: + Type: CommaDelimitedList + PublicCidrBlocks: + Type: CommaDelimitedList +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + PublicSubnet: + Type: LocalModule + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + PrivateSubnet: + Type: LocalModule + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex +``` + +""" + +# pylint: disable=fixme,too-many-instance-attributes + +import copy +import logging +import os +from collections import OrderedDict + +from awscli.customizations.cloudformation import exceptions +from awscli.customizations.cloudformation import yamlhelper + +from awscli.customizations.cloudformation.parse_sub import WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub + +LOG = logging.getLogger(__name__) + +RESOURCES = "Resources" +METADATA = "Metadata" +OVERRIDES = "Overrides" +DEPENDSON = "DependsOn" +PROPERTIES = "Properties" +CREATIONPOLICY = "CreationPolicy" +UPDATEPOLICY = "UpdatePolicy" +DELETIONPOLICY = "DeletionPolicy" +UPDATEREPLACEPOLICY = "UpdateReplacePolicy" +CONDITION = "Condition" +DEFAULT = "Default" +NAME = "Name" +SOURCE = "Source" +REF = "Ref" +SUB = "Fn::Sub" +GETATT = "Fn::GetAtt" +PARAMETERS = "Parameters" +MODULES = "Modules" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" +OUTPUTS = "Outputs" +MAP = "Map" +MAP_PLACEHOLDER = "$MapValue" +INDEX_PLACEHOLDER = "$MapIndex" + + +def process_module_section(template, base_path, parent_path): + "Recursively process the Modules section of a template" + if MODULES in template: + if not isdict(template[MODULES]): + msg = "Modules section is invalid" + raise exceptions.InvalidModuleError(msg=msg) + + # Process each Module node separately + for k, v in template[MODULES].items(): + module = make_module(template, k, v, base_path, parent_path) + template = module.process() + + # Remove the Modules section from the template + del template[MODULES] + + return template + + +def make_module(template, name, config, base_path, parent_path): + "Create an instance of a module based on a template and the module config" + module_config = {} + module_config[NAME] = name + if SOURCE not in config: + msg = f"{name} missing {SOURCE}" + raise exceptions.InvalidModulePathError(msg=msg) + relative_path = config[SOURCE] + module_config[SOURCE] = os.path.join(base_path, relative_path) + module_config[SOURCE] = os.path.normpath(module_config[SOURCE]) + if module_config[SOURCE] == parent_path: + msg = f"Module refers to itself: {parent_path}" + raise exceptions.InvalidModuleError(msg=msg) + if PROPERTIES in config: + module_config[PROPERTIES] = config[PROPERTIES] + if OVERRIDES in config: + module_config[OVERRIDES] = config[OVERRIDES] + return Module(template, module_config) + + +def map_placeholders(i, token, val): + "Replace $MapValue and $MapIndex" + if SUB in val: + sub = val[SUB] + r = sub.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + words = parse_sub(r) + need_sub = False + for word in words: + if word.t != WordType.STR: + need_sub = True + break + if need_sub: + return {SUB: r} + return r + r = val.replace(MAP_PLACEHOLDER, token) + r = r.replace(INDEX_PLACEHOLDER, f"{i}") + return r + + +# pylint: disable=too-many-locals,too-many-nested-blocks +def process_resources_section(template, base_path, parent_path, parent_module): + "Recursively process the Resources section of the template" + if parent_module is None: + # Make a fake Module instance to handle find_ref for Maps + # The only valid way to do this at the template level + # is to specify a default for a Parameter, since we need to + # resolve the actual value client-side + parent_module = Module(template, {NAME: "", SOURCE: ""}) + if PARAMETERS in template: + parent_module.params = template[PARAMETERS] + + for k, v in template[RESOURCES].copy().items(): + if TYPE in v and v[TYPE] == LOCAL_MODULE: + # First, pre-process local modules that are looping over a list + if MAP in v: + # Expect Map to be a CSV or ref to a CSV + m = v[MAP] + if isdict(m) and REF in m: + if parent_module is None: + msg = "Map is only valid in a module" + raise exceptions.InvalidModuleError(msg=msg) + m = parent_module.find_ref(m[REF]) + if m is None: + msg = f"{k} has an invalid Map Ref" + raise exceptions.InvalidModuleError(msg=msg) + tokens = m.split(",") # TODO - use an actual csv parser? + for i, token in enumerate(tokens): + # Make a new resource + logical_id = f"{k}{i}" + resource = copy.deepcopy(v) + del resource[MAP] + # Replace $Map and $Index placeholders + for prop, val in resource[PROPERTIES].copy().items(): + resource[PROPERTIES][prop] = map_placeholders( + i, token, val + ) + template[RESOURCES][logical_id] = resource + + del template[RESOURCES][k] + + # Start over after pre-processing maps + for k, v in template[RESOURCES].copy().items(): + if TYPE in v and v[TYPE] == LOCAL_MODULE: + module = make_module(template, k, v, base_path, parent_path) + template = module.process() + del template[RESOURCES][k] + return template + + +def isdict(v): + "Returns True if the type is a dict or OrderedDict" + return isinstance(v, (dict, OrderedDict)) + + +def read_source(source): + "Read the source file and return the content as a string" + + if not isinstance(source, str) or not os.path.isfile(source): + raise exceptions.InvalidModulePathError(source=source) + + with open(source, "r", encoding="utf-8") as s: + return s.read() + + +def merge_props(original, overrides): + """ + This function merges dicts, replacing values in the original with + overrides. This function is recursive and can act on lists and scalars. + See the unit tests for example merges. + See tests/unit/customizations/cloudformation/modules/policy-*.yaml + + :return A new value with the overridden properties + """ + original_type = type(original) + override_type = type(overrides) + if not isdict(overrides) and override_type is not list: + return overrides + + if original_type is not override_type: + return overrides + + if isdict(original): + retval = original.copy() + for k in original: + if k in overrides: + retval[k] = merge_props(retval[k], overrides[k]) + for k in overrides: + if k not in original: + retval[k] = overrides[k] + return retval + + # original and overrides are lists + new_list = [] + for item in original: + new_list.append(item) + for item in overrides: + new_list.append(item) + return new_list + + +class Module: + """ + Process client-side modules. + + """ + + def __init__(self, template, module_config): + """ + Initialize the module with values from the parent template + + :param template The parent template dictionary + :param module_config The configuration from the parent Modules section + """ + + # The parent template dictionary + self.template = template + if RESOURCES not in self.template: + # The parent might only have Modules + self.template[RESOURCES] = {} + + # The name of the module, which is used as a logical id prefix + self.name = module_config[NAME] + + # The location of the source for the module, a URI string + self.source = module_config[SOURCE] + + # The Properties from the parent template + self.props = {} + if PROPERTIES in module_config: + self.props = module_config[PROPERTIES] + + # The Overrides from the parent template + self.overrides = {} + if OVERRIDES in module_config: + self.overrides = module_config[OVERRIDES] + + # Resources defined in the module + self.resources = {} + + # Parameters defined in the module + self.params = {} + + # Outputs defined in the module + self.outputs = {} + + def __str__(self): + "Print out a string with module details for logs" + return ( + f"module name: {self.name}, " + + f"source: {self.source}, props: {self.props}" + ) + + def process(self): + """ + Read the module source process it. + + :return: The modified parent template dictionary + """ + + content = read_source(self.source) + + module_dict = yamlhelper.yaml_parse(content) + if RESOURCES not in module_dict: + # The module may only have sub modules in the Modules section + self.resources = {} + else: + self.resources = module_dict[RESOURCES] + + if PARAMETERS in module_dict: + self.params = module_dict[PARAMETERS] + + if OUTPUTS in module_dict: + self.outputs = module_dict[OUTPUTS] + + # Recurse on nested modules + bp = os.path.dirname(self.source) + section = "" + try: + section = MODULES + process_module_section(module_dict, bp, self.source) + section = RESOURCES + process_resources_section(module_dict, bp, self.source, self) + except Exception as e: + msg = f"Failed to process {section} section: {e}" + LOG.exception(msg) + raise exceptions.InvalidModuleError(msg=msg) + + self.validate_overrides() + + for logical_id, resource in self.resources.items(): + self.process_resource(logical_id, resource) + + self.process_outputs() + + return self.template + + def process_outputs(self): + """ + Fix parent template output references. + + In the parent you can !GetAtt ModuleName.OutputName + This will be converted so that it's correct in the packaged template. + + Recurse over all sections in the parent template looking for + GetAtts and Subs that reference a module output value. + """ + sections = [RESOURCES, OUTPUTS] # TODO: Any others? + for section in sections: + if section not in self.template: + continue + for k, v in self.template[section].items(): + self.resolve_outputs(k, v, self.template, section) + + def resolve_outputs(self, k, v, d, n): + """ + Recursively resolve GetAtts and Subs that reference module outputs. + + :param name The name of the output + :param output The output dict + :param k The name of the node + :param v The value of the node + :param d The dict that holds the parent of k + :param n The name of the node that holds k + + If a reference is found, this function sets the value of d[n] + """ + if k == SUB: + self.resolve_output_sub(v, d, n) + elif k == GETATT: + self.resolve_output_getatt(v, d, n) + else: + if isdict(v): + for k2, v2 in v.copy().items(): + self.resolve_outputs(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + for k3, v3 in v2.copy().items(): + self.resolve_outputs(k3, v3, v, idx) + + def resolve_output_sub(self, v, d, n): + "Resolve a Sub that refers to a module output" + words = parse_sub(v, True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # A reference to an output has to be a getatt + resolved = "${" + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + # !Sub ${Content.BucketArn} -> !Sub ${ContentBucket.Arn} + if tokens[0] == self.name and tokens[1] in self.outputs: + output = self.outputs[tokens[1]] + if GETATT in output: + getatt = output[GETATT] + resolved = "${" + self.name + ".".join(getatt) + "}" + elif SUB in output: + resolved = "${" + self.name + output[SUB] + "}" + sub += resolved + + d[n] = {SUB: sub} + + # pylint:disable=too-many-branches + def resolve_output_getatt(self, v, d, n): + "Resolve a GetAtt that refers to a module output" + if not isinstance(v, list) or len(v) < 2: + msg = f"GetAtt {v} invalid" + raise exceptions.InvalidModuleError(msg=msg) + if v[0] == self.name and v[1] in self.outputs: + output = self.outputs[v[1]] + if GETATT in output: + getatt = output[GETATT] + if len(getatt) < 2: + msg = f"GetAtt {getatt} in Output {v[1]} is invalid" + raise exceptions.InvalidModuleError(msg=msg) + d[n] = {GETATT: [self.name + getatt[0], getatt[1]]} + elif SUB in output: + # Parse the Sub in the module output + words = parse_sub(output[SUB], True) + sub = "" + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + elif word.t == WordType.REF: + # This is a ref to a param or resource + # TODO: If it's a ref to a param...? is this allowed? + # If it's a resource, concatenante the name + resolved = "${" + word.w + "}" + if word.w in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + elif word.t == WordType.GETATT: + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} unexpected length" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + resolved = "${" + self.name + word.w + "}" + sub += resolved + + d[n] = {SUB: sub} + + def validate_overrides(self): + "Make sure resources referenced by overrides actually exist" + for logical_id in self.overrides: + if logical_id not in self.resources: + msg = f"Override {logical_id} not found in {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + + def process_resource(self, logical_id, resource): + "Process a single resource" + + # For each property (and property-like attribute), + # replace the value if it appears in parent overrides. + attrs = [ + PROPERTIES, + CREATIONPOLICY, + METADATA, + UPDATEPOLICY, + DELETIONPOLICY, + CONDITION, + UPDATEREPLACEPOLICY, + DEPENDSON, + ] + for a in attrs: + self.process_overrides(logical_id, resource, a) + + # Resolve refs, subs, and getatts + # (Process module Parameters and parent Properties) + container = {} + # We need the container for the first iteration of the recursion + container[RESOURCES] = self.resources + self.resolve(logical_id, resource, container, RESOURCES) + + self.template[RESOURCES][self.name + logical_id] = resource + + def process_overrides(self, logical_id, resource, attr_name): + """ + Replace overridden values in a property-like attribute of a resource. + + (Properties, Metadata, CreationPolicy, and UpdatePolicy) + + Overrides are a way to customize modules without needing a Parameter. + + Example template.yaml: + + Modules: + Foo: + Source: ./module.yaml + Overrides: + Bar: + Properties: + Name: bbb + + Example module.yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: aaa + + Output yaml: + + Resources: + Bar: + Type: A::B::C + Properties: + Name: bbb + """ + + if logical_id not in self.overrides: + return + + resource_overrides = self.overrides[logical_id] + if attr_name not in resource_overrides: + return + + # Might be overriding something that's not in the module at all, + # like a Bucket with no Properties + if attr_name not in resource: + if attr_name in resource_overrides: + resource[attr_name] = resource_overrides[attr_name] + else: + return + + original = resource[attr_name] + overrides = resource_overrides[attr_name] + resource[attr_name] = merge_props(original, overrides) + + def resolve(self, k, v, d, n): + """ + Resolve Refs, Subs, and GetAtts recursively. + + :param k The name of the node + :param v The value of the node + :param d The dict that is the parent of the dict that holds k, v + :param n The name of the dict that holds k, v + + Example + + Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + + In the above example, + k = !Ref, v = Name, d = Properties{}, n = BucketName + + So we can set d[n] = resolved_value (which replaces {k,v}) + + In the prior iteration, + k = BucketName, v = {!Ref, Name}, d = Bucket{}, n = Properties + """ + + # print(f"k: {k}, v: {v}, d: {d}, n: {n}") + + if k == REF: + self.resolve_ref(v, d, n) + elif k == SUB: + self.resolve_sub(v, d, n) + elif k == GETATT: + self.resolve_getatt(v, d, n) + else: + if isdict(v): + vc = v.copy() + for k2, v2 in vc.items(): + self.resolve(k2, v2, d[n], k) + elif isinstance(v, list): + idx = -1 + for v2 in v: + idx = idx + 1 + if isdict(v2): + v2c = v2.copy() + for k3, v3 in v2c.items(): + self.resolve(k3, v3, v, idx) + + def resolve_ref(self, v, d, n): + """ + Look for the Ref in the parent template Properties if it matches + a module Parameter name. If it's not there, use the default if + there is one. If not, raise an error. + + If there is no matching Parameter, look for a resource with that + name in this module and fix the logical id so it has the prefix. + + Otherwise just leave it be and assume the module author is + expecting the parent template to have that Reference. + """ + if not isinstance(v, str): + msg = f"Ref should be a string: {v}" + raise exceptions.InvalidModuleError(msg=msg) + + found = self.find_ref(v) + if found is not None: + d[n] = found + + # pylint: disable=too-many-branches,unused-argument + def resolve_sub(self, v, d, n): + """ + Parse the Sub string and break it into tokens. + + If we can fully resolve it, we can replace it with a string. + + Use the same logic as with resolve_ref. + """ + # print(f"resolve_sub v: {v}, d: {d}, n: {n}") + words = parse_sub(v, True) + sub = "" + need_sub = False + for word in words: + if word.t == WordType.STR: + sub += word.w + elif word.t == WordType.AWS: + sub += "${AWS::" + word.w + "}" + need_sub = True + elif word.t == WordType.REF: + resolved = "${" + word.w + "}" + need_sub = True + found = self.find_ref(word.w) + if found is not None: + if isinstance(found, str): + need_sub = False + resolved = found + else: + if REF in found: + resolved = "${" + found[REF] + "}" + elif SUB in found: + resolved = found[SUB] + sub += resolved + elif word.t == WordType.GETATT: + need_sub = True + resolved = "${" + word.w + "}" + tokens = word.w.split(".", 1) + if len(tokens) < 2: + msg = f"GetAtt {word.w} has unexpected number of tokens" + raise exceptions.InvalidModuleError(msg=msg) + if tokens[0] in self.resources: + tokens[0] = self.name + tokens[0] + resolved = "${" + tokens[0] + "." + tokens[1] + "}" + sub += resolved + + if need_sub: + d[n] = {SUB: sub} + else: + d[n] = sub + + def resolve_getatt(self, v, d, n): + """ + Resolve a GetAtt. All we do here is add the prefix. + + !GetAtt Foo.Bar becomes !GetAtt ModuleNameFoo.Bar + """ + if not isinstance(v, list): + msg = f"GetAtt {v} is not a list" + raise exceptions.InvalidModuleError(msg=msg) + logical_id = self.name + v[0] + d[n] = {GETATT: [logical_id, v[1]]} + + def find_ref(self, name): + """ + Find a Ref. + + A Ref might be to a module Parameter with a matching parent + template Property, or a Parameter Default. It could also + be a reference to another resource in this module. + + :param name The name to search for + :return The referenced element or None + """ + # print(f"find_ref {name}, props: {self.props}, params: {self.params}") + if name in self.props: + if name not in self.params: + # The parent tried to set a property that doesn't exist + # in the Parameters section of this module + msg = f"{name} not found in module Parameters: {self.source}" + raise exceptions.InvalidModuleError(msg=msg) + return self.props[name] + + if name in self.params: + param = self.params[name] + if DEFAULT in param: + # Use the default value of the Parameter + return param[DEFAULT] + msg = f"{name} does not have a Default and is not a Property" + raise exceptions.InvalidModuleError(msg=msg) + + for logical_id in self.resources: + if name == logical_id: + # Simply rename local references to include the module name + return {REF: self.name + logical_id} + + return None diff --git a/awscli/customizations/cloudformation/parse_sub.py b/awscli/customizations/cloudformation/parse_sub.py new file mode 100644 index 000000000000..f096d70b4812 --- /dev/null +++ b/awscli/customizations/cloudformation/parse_sub.py @@ -0,0 +1,130 @@ +""" +This module parses CloudFormation Sub strings. + +For example: + + !Sub abc-${AWS::Region}-def-${Foo} + +The string is broken down into "words" that are one of four types: + + String: A literal string component + Ref: A reference to another resource or paramter like ${Foo} + AWS: An AWS pseudo-parameter like ${AWS::Region} + GetAtt: A reference to an attribute like ${Foo.Bar} +""" +#pylint: disable=too-few-public-methods + +from enum import Enum + +DATA = ' ' # Any other character +DOLLAR = '$' +OPEN = '{' +CLOSE = '}' +BANG = '!' +SPACE = ' ' + +class WordType(Enum): + "Word type enumeration" + STR = 0 # A literal string fragment + REF = 1 # ${ParamOrResourceName} + AWS = 2 # ${AWS::X} + GETATT = 3 # ${X.Y} + +class State(Enum): + "State machine enumeration" + READSTR = 0 + READVAR = 1 + MAYBE = 2 + READLIT = 3 + +class SubWord: + "A single word with a type and the word itself" + def __init__(self, word_type, word): + self.t = word_type + self.w = word # Does not include the ${} if it's not a STR + + def __str__(self): + return f"{self.t} {self.w}" + +#pylint: disable=too-many-branches,too-many-statements +def parse_sub(sub_str, leave_bang=False): + """ + Parse a Sub string + + :param leave_bang If this is True, leave the ! in literals + :return list of words + """ + words = [] + state = State.READSTR + buf = '' + last = '' + for i, char in enumerate(sub_str): + if char == DOLLAR: + if state != State.READVAR: + state = State.MAYBE + else: + # This is a literal $ inside a variable: "${AB$C}" + buf += char + elif char == OPEN: + if state == State.MAYBE: + # Peek to see if we're about to start a LITERAL ! + if len(sub_str) > i+1 and sub_str[i+1] == BANG: + # Treat this as part of the string, not a var + buf += "${" + state = State.READLIT + else: + state = State.READVAR + # We're about to start reading a variable. + # Append the last word in the buffer if it's not empty + if buf: + words.append(SubWord(WordType.STR, buf)) + buf = '' + else: + buf += char + elif char == CLOSE: + if state == State.READVAR: + # Figure out what type it is + if buf.startswith("AWS::"): + word_type = WordType.AWS + elif '.' in buf: + word_type = WordType.GETATT + else: + word_type = WordType.REF + buf = buf.replace("AWS::", "", 1) + words.append(SubWord(word_type, buf)) + buf = '' + state = State.READSTR + else: + buf += char + elif char == BANG: + # ${!LITERAL} becomes ${LITERAL} + if state == State.READLIT: + # Don't write the ! to the string + state = State.READSTR + if leave_bang: + # Unless we actually want it + buf += char + else: + # This is a ! somewhere not related to a LITERAL + buf += char + elif char == SPACE: + # Ignore spaces around Refs. ${ ABC } == ${ABC} + if state != State.READVAR: + buf += char + else: + if state == State.MAYBE: + buf += last # Put the $ back on the buffer + state = State.READSTR + buf += char + + last = char + + if buf: + words.append(SubWord(WordType.STR, buf)) + + # Handle malformed strings, like "ABC${XYZ" + if state != State.READSTR: + # Ended the string in the middle of a variable? + raise ValueError("invalid string, unclosed variable") + + return words diff --git a/tests/unit/customizations/cloudformation/modules/basic-expect.yaml b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml new file mode 100644 index 000000000000..1010e68cf134 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-expect.yaml @@ -0,0 +1,16 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/basic-module.yaml b/tests/unit/customizations/cloudformation/modules/basic-module.yaml new file mode 100644 index 000000000000..cbb5d4bbd90b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-module.yaml @@ -0,0 +1,14 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something diff --git a/tests/unit/customizations/cloudformation/modules/basic-template.yaml b/tests/unit/customizations/cloudformation/modules/basic-template.yaml new file mode 100644 index 000000000000..712eb2a8a87b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/basic-template.yaml @@ -0,0 +1,12 @@ +Modules: + Content: + Source: ./basic-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def +Resources: + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/map-expect.yaml b/tests/unit/customizations/cloudformation/modules/map-expect.yaml new file mode 100644 index 000000000000..69b789b9a716 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-expect.yaml @@ -0,0 +1,17 @@ +Parameters: + List: + Type: CommaDelimitedList + Default: A,B,C +Resources: + Content0Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-A + Content1Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-B + Content2Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: my-bucket-C diff --git a/tests/unit/customizations/cloudformation/modules/map-module.yaml b/tests/unit/customizations/cloudformation/modules/map-module.yaml new file mode 100644 index 000000000000..e96d93d1733e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-module.yaml @@ -0,0 +1,9 @@ +Parameters: + Name: + Type: String + +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name diff --git a/tests/unit/customizations/cloudformation/modules/map-template.yaml b/tests/unit/customizations/cloudformation/modules/map-template.yaml new file mode 100644 index 000000000000..ba6a67dcd30d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/map-template.yaml @@ -0,0 +1,12 @@ +Parameters: + List: + Type: CommaDelimitedList + Default: A,B,C + +Resources: + Content: + Type: LocalModule + Source: ./map-module.yaml + Map: !Ref List + Properties: + Name: !Sub my-bucket-$MapValue diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml new file mode 100644 index 000000000000..561947b750a2 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-expect.yaml @@ -0,0 +1,14 @@ +Parameters: + ParentVal: + Type: String + AppName: + Type: String +Resources: + MySubBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: + Fn::Sub: ${ParentVal}-abc + YName: + Fn::Sub: ${AppName}-xyz diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml new file mode 100644 index 000000000000..208307bd37f5 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-module.yaml @@ -0,0 +1,11 @@ +Parameters: + AppName: + Type: String +Resources: + Sub: + Type: LocalModule + Source: "./modinmod-submodule.yaml" + Properties: + X: !Sub ${ParentVal}-abc + Y: !Sub ${AppName}-xyz + diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml new file mode 100644 index 000000000000..745292ee0711 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-submodule.yaml @@ -0,0 +1,12 @@ +Parameters: + X: + Type: String + Y: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: mod-in-mod-bucket + XName: !Ref X + YName: !Ref Y diff --git a/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml new file mode 100644 index 000000000000..99cd480b512d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/modinmod-template.yaml @@ -0,0 +1,11 @@ +Parameters: + ParentVal: + Type: String + AppName: + Type: String +Resources: + My: + Type: LocalModule + Source: ./modinmod-module.yaml + Properties: + AppName: !Ref AppName diff --git a/tests/unit/customizations/cloudformation/modules/output-expect.yaml b/tests/unit/customizations/cloudformation/modules/output-expect.yaml new file mode 100644 index 000000000000..a4ebb8fe8ea5 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-expect.yaml @@ -0,0 +1,17 @@ +Outputs: + ExampleOutput: + Value: + Fn::GetAtt: + - ContentBucket + - Arn + ExampleSub: + Value: + Fn::Sub: ${ContentBucket.Arn} + ExampleGetSub: + Value: + Fn::Sub: ${ContentBucket.Arn} +Resources: + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo diff --git a/tests/unit/customizations/cloudformation/modules/output-module.yaml b/tests/unit/customizations/cloudformation/modules/output-module.yaml new file mode 100644 index 000000000000..f685ca154d84 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-module.yaml @@ -0,0 +1,12 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name +Outputs: + BucketArn: !GetAtt Bucket.Arn + BucketArnSub: !Sub ${Bucket.Arn} + diff --git a/tests/unit/customizations/cloudformation/modules/output-template.yaml b/tests/unit/customizations/cloudformation/modules/output-template.yaml new file mode 100644 index 000000000000..872d3423597b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/output-template.yaml @@ -0,0 +1,12 @@ +Modules: + Content: + Source: ./output-module.yaml + Properties: + Name: foo +Outputs: + ExampleOutput: + Value: !GetAtt Content.BucketArn + ExampleSub: + Value: !Sub ${Content.BucketArn} + ExampleGetSub: + Value: !GetAtt Content.BucketArnSub diff --git a/tests/unit/customizations/cloudformation/modules/policy-expect.yaml b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml new file mode 100644 index 000000000000..8d524c1fb4ce --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-expect.yaml @@ -0,0 +1,16 @@ +Resources: + AccessPolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - arn:aws:s3:::foo + - arn:aws:s3:::foo/* + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar + PolicyName: my-policy + RoleName: foo diff --git a/tests/unit/customizations/cloudformation/modules/policy-module.yaml b/tests/unit/customizations/cloudformation/modules/policy-module.yaml new file mode 100644 index 000000000000..7de41d792584 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-module.yaml @@ -0,0 +1,19 @@ +Parameters: + Role: + Type: String + BucketName: + Default: foo +Resources: + Policy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyDocument: + Statement: + - Effect: ALLOW + Action: s3:List* + Resource: + - !Sub arn:aws:s3:::${BucketName} + - !Sub arn:aws:s3:::${BucketName}/* + PolicyName: my-policy + RoleName: !Ref Role + diff --git a/tests/unit/customizations/cloudformation/modules/policy-template.yaml b/tests/unit/customizations/cloudformation/modules/policy-template.yaml new file mode 100644 index 000000000000..9f587f7bda9d --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/policy-template.yaml @@ -0,0 +1,13 @@ +Modules: + Access: + Source: ./policy-module.yaml + Properties: + Role: foo + Overrides: + Policy: + Properties: + PolicyDocument: + Statement: + - Effect: DENY + Action: s3:Put* + Resource: arn:aws:s3:::bar diff --git a/tests/unit/customizations/cloudformation/modules/sub-expect.yaml b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml new file mode 100644 index 000000000000..79cb95080ccc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-expect.yaml @@ -0,0 +1,21 @@ +Resources: + MyBucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + X: + Fn::Sub: ${Foo} + Y: + - Fn::Sub: noparent0-${Foo} + - Fn::Sub: noparent1-${Foo} + Z: + - Fn::Sub: ${Foo} + - Ref: MyBucket2 + - ZZ: + ZZZ: + ZZZZ: + Fn::Sub: ${Foo} + MyBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/sub-module.yaml b/tests/unit/customizations/cloudformation/modules/sub-module.yaml new file mode 100644 index 000000000000..8d4937f4aefc --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-module.yaml @@ -0,0 +1,23 @@ +Parameters: + Name: + Type: String + SubName: + Type: String +Resources: + Bucket1: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub ${Name} + X: !Sub "${SubName}" + Y: + - !Sub noparent0-${SubName} + - !Sub noparent1-${SubName} + Z: + - !Ref SubName + - !Ref Bucket2 + - ZZ: + ZZZ: + ZZZZ: !Sub "${SubName}" + Bucket2: + Type: AWS::S3::Bucket + diff --git a/tests/unit/customizations/cloudformation/modules/sub-template.yaml b/tests/unit/customizations/cloudformation/modules/sub-template.yaml new file mode 100644 index 000000000000..feeb263bfcad --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/sub-template.yaml @@ -0,0 +1,11 @@ +Resources: + My: + Type: LocalModule + Source: ./sub-module.yaml + Properties: + Name: foo + SubName: !Sub ${Foo} + Overrides: + Bucket2: + Properties: + BucketName: bar diff --git a/tests/unit/customizations/cloudformation/modules/subnet-module.yaml b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml new file mode 100644 index 000000000000..2291bd6b0c7e --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/subnet-module.yaml @@ -0,0 +1,59 @@ +Parameters: + + AZSelection: + Type: Number + + SubnetCidrBlock: + Type: String + +Resources: + + Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: !Select [!Ref AZSelection, Fn::GetAZs: !Ref "AWS::Region"] + CidrBlock: !Ref SubnetCidrBlock + MapIpOnLaunch: true + VpcId: !Ref VPC + + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + + RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref SubnetRouteTable + SubnetId: !Ref Subnet + + DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + RouteTableId: !Ref SubnetRouteTable + DependsOn: VPCGW + + EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + + NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: !GetAtt SubnetEIP.AllocationId + SubnetId: !Ref Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + diff --git a/tests/unit/customizations/cloudformation/modules/type-expect.yaml b/tests/unit/customizations/cloudformation/modules/type-expect.yaml new file mode 100644 index 000000000000..1010e68cf134 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-expect.yaml @@ -0,0 +1,16 @@ +Resources: + OtherResource: + Type: AWS::S3::Bucket + ContentBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: foo + OverrideMe: def + SubName: foo + ContentBucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::GetAtt: + - ContentBucket + - Something diff --git a/tests/unit/customizations/cloudformation/modules/type-module.yaml b/tests/unit/customizations/cloudformation/modules/type-module.yaml new file mode 100644 index 000000000000..cbb5d4bbd90b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-module.yaml @@ -0,0 +1,14 @@ +Parameters: + Name: + Type: String +Resources: + Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref Name + OverrideMe: abc + SubName: !Sub ${Name} + Bucket2: + Type: AWS::S3::Bucket + Properties: + BucketName: !GetAtt Bucket.Something diff --git a/tests/unit/customizations/cloudformation/modules/type-template.yaml b/tests/unit/customizations/cloudformation/modules/type-template.yaml new file mode 100644 index 000000000000..e868af732938 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/type-template.yaml @@ -0,0 +1,12 @@ +Resources: + Content: + Type: LocalModule + Source: ./type-module.yaml + Properties: + Name: foo + Overrides: + Bucket: + Properties: + OverrideMe: def + OtherResource: + Type: AWS::S3::Bucket diff --git a/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml new file mode 100644 index 000000000000..0d43ced5bd2b --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-expect.yaml @@ -0,0 +1,236 @@ +Resources: + NetworkVPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + NetworkPublicSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.128.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet0Subnet + NetworkPublicSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPublicSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.192.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPublicSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPublicSubnet1Subnet + NetworkPublicSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPublicSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPublicSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPublicSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPublicSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet0Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '0' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.0.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet0RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet0RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + NetworkPrivateSubnet0DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet0EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet0NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet0SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet0Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation + NetworkPrivateSubnet1Subnet: + Type: AWS::EC2::Subnet + Metadata: + guard: + SuppressedRules: + - SUBNET_AUTO_ASSIGN_PUBLIC_IP_DISABLED + Properties: + AvailabilityZone: + Fn::Select: + - '1' + - Fn::GetAZs: + Ref: AWS::Region + CidrBlock: 10.0.64.0/18 + MapIpOnLaunch: true + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: NetworkVPC + NetworkPrivateSubnet1RouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: + Ref: SubnetRouteTable + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + NetworkPrivateSubnet1DefaultRoute: + Type: AWS::EC2::Route + Metadata: + guard: + SuppressedRules: + - NO_UNRESTRICTED_ROUTE_TO_IGW + Properties: + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + RouteTableId: + Ref: SubnetRouteTable + DependsOn: VPCGW + NetworkPrivateSubnet1EIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + NetworkPrivateSubnet1NATGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NetworkPrivateSubnet1SubnetEIP + - AllocationId + SubnetId: + Ref: NetworkPrivateSubnet1Subnet + DependsOn: + - SubnetDefaultRoute + - SubnetRouteTableAssociation diff --git a/tests/unit/customizations/cloudformation/modules/vpc-module.yaml b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml new file mode 100644 index 000000000000..a33c4e29aada --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-module.yaml @@ -0,0 +1,37 @@ +Parameters: + + CidrBlock: + Type: String + + PrivateCidrBlocks: + Type: CommaDelimitedList + + PublicCidrBlocks: + Type: CommaDelimitedList + +Resources: + + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + InstanceTenancy: default + + PublicSubnet: + Type: LocalModule + Map: !Ref PublicCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + + PrivateSubnet: + Type: LocalModule + Map: !Ref PrivateCidrBlocks + Source: ./subnet-module.yaml + Properties: + SubnetCidrBlock: $MapValue + AZSelection: $MapIndex + diff --git a/tests/unit/customizations/cloudformation/modules/vpc-template.yaml b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml new file mode 100644 index 000000000000..92e512b57d67 --- /dev/null +++ b/tests/unit/customizations/cloudformation/modules/vpc-template.yaml @@ -0,0 +1,9 @@ +Resources: + + Network: + Type: LocalModule + Source: ./vpc-module.yaml + Properties: + CidrBlock: 10.0.0.0/16 + PrivateCidrBlocks: 10.0.0.0/18,10.0.64.0/18 + PublicCidrBlocks: 10.0.128.0/18,10.0.192.0/18 diff --git a/tests/unit/customizations/cloudformation/test_modules.py b/tests/unit/customizations/cloudformation/test_modules.py new file mode 100644 index 000000000000..0fa4fa12d880 --- /dev/null +++ b/tests/unit/customizations/cloudformation/test_modules.py @@ -0,0 +1,108 @@ +"Tests for module support in the package command" + +# pylint: disable=fixme + +from awscli.testutils import unittest +from awscli.customizations.cloudformation import yamlhelper +from awscli.customizations.cloudformation import modules +from awscli.customizations.cloudformation.parse_sub import SubWord, WordType +from awscli.customizations.cloudformation.parse_sub import parse_sub + +MODULES = "Modules" +RESOURCES = "Resources" +TYPE = "Type" +LOCAL_MODULE = "LocalModule" + + +class TestPackageModules(unittest.TestCase): + "Module tests" + + def setUp(self): + "Initialize the tests" + + def test_parse_sub(self): + "Test the parse_sub function" + cases = { + "ABC": [SubWord(WordType.STR, "ABC")], + "ABC-${XYZ}-123": [ + SubWord(WordType.STR, "ABC-"), + SubWord(WordType.REF, "XYZ"), + SubWord(WordType.STR, "-123"), + ], + "ABC-${!Literal}-1": [SubWord(WordType.STR, "ABC-${Literal}-1")], + "${ABC}": [SubWord(WordType.REF, "ABC")], + "${ABC.XYZ}": [SubWord(WordType.GETATT, "ABC.XYZ")], + "ABC${AWS::AccountId}XYZ": [ + SubWord(WordType.STR, "ABC"), + SubWord(WordType.AWS, "AccountId"), + SubWord(WordType.STR, "XYZ"), + ], + "BAZ${ABC$XYZ}FOO$BAR": [ + SubWord(WordType.STR, "BAZ"), + SubWord(WordType.REF, "ABC$XYZ"), + SubWord(WordType.STR, "FOO$BAR"), + ], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + "${ ABC }": [SubWord(WordType.REF, "ABC")], + " ABC ": [SubWord(WordType.STR, " ABC ")], + } + + for sub, expect in cases.items(): + words = parse_sub(sub, False) + self.assertEqual( + len(expect), + len(words), + f'"{sub}": words len is {len(words)}, expected {len(expect)}', + ) + for i, w in enumerate(expect): + self.assertEqual( + words[i].t, w.t, f'"{sub}": got {words[i]}, expected {w}' + ) + self.assertEqual( + words[i].w, w.w, f'"{sub}": got {words[i]}, expected {w}' + ) + + # Invalid strings should fail + sub = "${AAA" + with self.assertRaises(Exception, msg=f'"{sub}": should have failed'): + parse_sub(sub, False) + + def test_merge_props(self): + "Test the merge_props function" + + original = {"b": "c", "d": {"e": "f", "i": [1, 2, 3]}} + overrides = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [4, 5]}} + expect = {"b": "cc", "d": {"e": "ff", "g": "h", "i": [1, 2, 3, 4, 5]}} + merged = modules.merge_props(original, overrides) + self.assertEqual(merged, expect) + + def test_main(self): + "Run tests on sample templates that include local modules" + + # The tests are in the modules directory. + # Each test has 3 files: + # test-template.yaml, test-module.yaml, and test-expect.yaml + tests = [ + "basic", + "type", + "sub", + "modinmod", + "output", + "policy", + "vpc", + "map", + ] + for test in tests: + base = "unit/customizations/cloudformation/modules" + t = modules.read_source(f"{base}/{test}-template.yaml") + td = yamlhelper.yaml_parse(t) + e = modules.read_source(f"{base}/{test}-expect.yaml") + + # Modules section + td = modules.process_module_section(td, base, t) + + # Resources with Type LocalModule + td = modules.process_resources_section(td, base, t, None) + + processed = yamlhelper.yaml_dump(td) + self.assertEqual(e, processed)