Molecule Destroy ephemeral EC2 instances

[matt-horwood-mayden]

Summary

when running molecule a molecule test, any instance not made with molecule is destroyed

Issue Type

Bug Report

Component Name

ec2_instance

Ansible Version

$ ansible --version
ansible [core 2.16.7]
  config file = /Users/matt.horwood/.ansible.cfg
  configured module search path = ['/Users/matt.horwood/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /Users/matt.horwood/git/dev-ops-ansible/environment/.env/lib/python3.11/site-packages/ansible
  ansible collection location = /Users/matt.horwood/.ansible/collections:/usr/share/ansible/collections:/Users/matt.horwood/git/dev-ops-ansible/collections
  executable location = /Users/matt.horwood/git/dev-ops-ansible/environment/.env/bin/ansible
  python version = 3.11.10 (main, Sep  7 2024, 01:03:31) [Clang 15.0.0 (clang-1500.3.9.4)] (/Users/matt.horwood/git/dev-ops-ansible/environment/.env/bin/python3.11)
  jinja version = 3.1.4
  libyaml = True

Collection Versions

$ ansible-galaxy collection list
# /Users/matt.horwood/.ansible/collections/ansible_collections
Collection                               Version
---------------------------------------- -------
amazon.aws                               8.1.0
community.docker                         3.11.0
community.general                        9.2.0
community.library_inventory_filtering_v1 1.0.1
community.mysql                          3.9.0

# /Users/matt.horwood/git/dev-ops-ansible/collections/ansible_collections
Collection                               Version
---------------------------------------- -------
mayden.library                           1.0.0
mayden.users                             1.0.0

# /Users/matt.horwood/git/dev-ops-ansible/environment/.env/lib/python3.11/site-packages/ansible_collections
Collection                               Version
---------------------------------------- -------
amazon.aws                               7.6.0
ansible.netcommon                        5.3.0
ansible.posix                            1.5.4
ansible.utils                            2.12.0
ansible.windows                          2.3.0
arista.eos                               6.2.2
awx.awx                                  23.9.0
azure.azcollection                       1.19.0
check_point.mgmt                         5.2.3
chocolatey.chocolatey                    1.5.1
cisco.aci                                2.9.0
cisco.asa                                4.0.3
cisco.dnac                               6.13.3
cisco.intersight                         2.0.9
cisco.ios                                5.3.0
cisco.iosxr                              6.1.1
cisco.ise                                2.9.1
cisco.meraki                             2.18.1
cisco.mso                                2.6.0
cisco.nxos                               5.3.0
cisco.ucs                                1.10.0
cloud.common                             2.1.4
cloudscale_ch.cloud                      2.3.1
community.aws                            7.2.0
community.azure                          2.0.0
community.ciscosmb                       1.0.9
community.crypto                         2.20.0
community.digitalocean                   1.26.0
community.dns                            2.9.1
community.docker                         3.10.1
community.general                        8.6.1
community.grafana                        1.8.0
community.hashi_vault                    6.2.0
community.hrobot                         1.9.2
community.library_inventory_filtering_v1 1.0.1
community.libvirt                        1.3.0
community.mongodb                        1.7.4
community.mysql                          3.9.0
community.network                        5.0.2
community.okd                            2.3.0
community.postgresql                     3.4.1
community.proxysql                       1.5.1
community.rabbitmq                       1.3.0
community.routeros                       2.15.0
community.sap                            2.0.0
community.sap_libs                       1.4.2
community.sops                           1.6.7
community.vmware                         4.4.0
community.windows                        2.2.0
community.zabbix                         2.4.0
containers.podman                        1.13.0
cyberark.conjur                          1.2.2
cyberark.pas                             1.0.25
dellemc.enterprise_sonic                 2.4.0
dellemc.openmanage                       8.7.0
dellemc.powerflex                        2.4.0
dellemc.unity                            1.7.1
f5networks.f5_modules                    1.28.0
fortinet.fortimanager                    2.5.0
fortinet.fortios                         2.3.6
frr.frr                                  2.0.2
gluster.gluster                          1.0.2
google.cloud                             1.3.0
grafana.grafana                          2.2.5
hetzner.hcloud                           2.5.0
hpe.nimble                               1.1.4
ibm.qradar                               2.1.0
ibm.spectrum_virtualize                  2.0.0
ibm.storage_virtualize                   2.3.1
infinidat.infinibox                      1.4.5
infoblox.nios_modules                    1.6.1
inspur.ispim                             2.2.1
inspur.sm                                2.3.0
junipernetworks.junos                    5.3.1
kaytus.ksmanage                          1.2.1
kubernetes.core                          2.4.2
lowlydba.sqlserver                       2.3.2
microsoft.ad                             1.5.0
netapp.aws                               21.7.1
netapp.azure                             21.10.1
netapp.cloudmanager                      21.22.1
netapp.elementsw                         21.7.0
netapp.ontap                             22.11.0
netapp.storagegrid                       21.12.0
netapp.um_info                           21.8.1
netapp_eseries.santricity                1.4.0
netbox.netbox                            3.18.0
ngine_io.cloudstack                      2.3.0
ngine_io.exoscale                        1.1.0
openstack.cloud                          2.2.0
openvswitch.openvswitch                  2.1.1
ovirt.ovirt                              3.2.0
purestorage.flasharray                   1.28.0
purestorage.flashblade                   1.17.0
purestorage.fusion                       1.6.1
sensu.sensu_go                           1.14.0
splunk.es                                2.1.2
t_systems_mms.icinga_director            2.0.1
telekom_mms.icinga_director              1.35.0
theforeman.foreman                       3.15.0
vmware.vmware_rest                       2.3.1
vultr.cloud                              1.12.1
vyos.vyos                                4.1.0
wti.remote                               1.0.5

AWS SDK versions

$ pip show boto boto3 botocore
Name: boto
Version: 2.49.0
Summary: Amazon Web Services Library
Home-page: https://github.com/boto/boto/
Author: Mitch Garnaat
Author-email: [email protected]
License: MIT
Location: /Users/matt.horwood/git/dev-ops-ansible/environment/.env/lib/python3.11/site-packages
Requires:
Required-by:
---
Name: boto3
Version: 1.35.14
Summary: The AWS SDK for Python
Home-page: https://github.com/boto/boto3
Author: Amazon Web Services
Author-email:
License: Apache License 2.0
Location: /Users/matt.horwood/git/dev-ops-ansible/environment/.env/lib/python3.11/site-packages
Requires: botocore, jmespath, s3transfer
Required-by:
---
Name: botocore
Version: 1.35.14
Summary: Low-level, data-driven core of boto 3.
Home-page: https://github.com/boto/botocore
Author: Amazon Web Services
Author-email:
License: Apache License 2.0
Location: /Users/matt.horwood/git/dev-ops-ansible/environment/.env/lib/python3.11/site-packages
Requires: jmespath, python-dateutil, urllib3
Required-by: boto3, s3transfer

Configuration

$ ansible-config dump --only-changed
ANSIBLE_COW_SELECTION(env: ANSIBLE_COW_SELECTION) = /Users/matt.horwood/git/cowsay/cows/cowfee.cow
ANSIBLE_FORCE_COLOR(/Users/matt.horwood/.ansible.cfg) = True
ANSIBLE_PIPELINING(/Users/matt.horwood/.ansible.cfg) = True
COLLECTIONS_PATHS(/Users/matt.horwood/.ansible.cfg) = ['/Users/matt.horwood/.ansible/collections', '/usr/share/ansible/collections', '/Users/matt.horwood/git/dev-ops-ansible/collections']
CONFIG_FILE() = /Users/matt.horwood/.ansible.cfg
DEFAULT_BECOME(/Users/matt.horwood/.ansible.cfg) = True
DEFAULT_BECOME_ASK_PASS(/Users/matt.horwood/.ansible.cfg) = True
DEFAULT_FORKS(/Users/matt.horwood/.ansible.cfg) = 15
DEFAULT_HOST_LIST(/Users/matt.horwood/.ansible.cfg) = ['/Users/matt.horwood/git/dev-ops-ansible/inventories/production']
DEFAULT_REMOTE_USER(/Users/matt.horwood/.ansible.cfg) = mayhealthv_mho
DEFAULT_ROLES_PATH(/Users/matt.horwood/.ansible.cfg) = ['/Users/matt.horwood/git/dev-ops-ansible/roles']
DEFAULT_TIMEOUT(/Users/matt.horwood/.ansible.cfg) = 60
DEFAULT_TRANSPORT(/Users/matt.horwood/.ansible.cfg) = ssh
DEFAULT_VAULT_PASSWORD_FILE(/Users/matt.horwood/.ansible.cfg) = /Users/matt.horwood/.vault
DISPLAY_ARGS_TO_STDOUT(/Users/matt.horwood/.ansible.cfg) = False
INTERPRETER_PYTHON(/Users/matt.horwood/.ansible.cfg) = /usr/bin/python3
INVENTORY_ENABLED(/Users/matt.horwood/.ansible.cfg) = ['aws_ec2', 'ini', 'yaml']
PAGER(env: PAGER) = less

OS / Environment

AWS CLI

Steps to Reproduce

---
- name: Destroy
  hosts: localhost
  connection: local
  gather_facts: false
  no_log: "{{ molecule_no_log }}"
  collections:
    - community.aws
  vars:
    # Run config handling
    default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}"
    default_run_config:
      run_id: "{{ default_run_id }}"

    run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml"
    run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}"
    run_config: '{{ default_run_config | combine(run_config_from_file) }}'

    # Platform settings handling
    default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}"
    default_key_inject_method: cloud-init # valid values: [cloud-init, ec2]
    default_key_name: "molecule-{{ run_config.run_id }}"
    default_security_group_name: "molecule-{{ run_config.run_id }}"

    platform_defaults:
      aws_profile: "{{ default_aws_profile }}"
      key_inject_method: "{{ default_key_inject_method }}"
      key_name: "{{ default_key_name }}"
      region: ""
      security_group_name: "{{ default_security_group_name }}"
      security_groups: []
      vpc_id: ""
      vpc_subnet_id: ""

    # Merging defaults into a list of dicts is, it turns out, not straightforward
    platforms: >-
      {{ [platform_defaults | dict2items]
           | product(molecule_yml.platforms | map('dict2items') | list)
           | map('flatten', levels=1)
           | list
           | map('items2dict')
           | list }}

    # Stored instance config
    instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '{}') | from_yaml }}"
  pre_tasks:
    - name: Validate platform configurations
      assert:
        that:
          - platforms | length > 0
          - platform.name is string and platform.name | length > 0
          - platform.aws_profile is string
          - platform.key_inject_method is in ["cloud-init", "ec2"]
          - platform.key_name is string and platform.key_name | length > 0
          - platform.region is string
          - platform.security_group_name is string and platform.security_group_name | length > 0
          - platform.security_groups is sequence
          - platform.vpc_id is string
          - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0
        quiet: true
      loop: '{{ platforms }}'
      loop_control:
        loop_var: platform
        label: "{{ platform.name }}"
  tasks:
    - name: Look up subnets to determine VPCs (if needed)
      ec2_vpc_subnet_info:
        subnet_ids: "{{ item.vpc_subnet_id }}"
      loop: "{{ platforms }}"
      loop_control:
        label: "{{ item.name }}"
      when: not item.vpc_id
      register: subnet_info

    - name: Validate discovered information
      assert:
        that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0)
        quiet: true
      loop: "{{ platforms }}"
      loop_control:
        loop_var: platform
        index_var: index
        label: "{{ platform.name }}"

    - name: Destroy ephemeral EC2 instances
      ec2_instance:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        instance_ids: "{{ instance_config | map(attribute='instance_ids') | flatten }}"
        state: absent
      loop: "{{ platforms }}"
      loop_control:
        label: "{{ item.name }}"
      register: ec2_instances_async

    - name: you dead yet
      ec2_instance_info:
        instance_ids: "{{ instance.instance_id }}"
      vars:
        instance: "{{ ec2_instances_async.results[0].instances[index] }}"
      loop: "{{ ec2_instances_async.results }}"
      loop_control:
        index_var: index
        label: "{{ platforms[index].name }}"
      until: instance.state.name == "terminated"
      when: ec2_instances_async is changed
      retries: 300

    - name: Write Molecule instance configs
      copy:
        dest: "{{ molecule_instance_config }}"
        content: "{{ {} | to_yaml }}"

    - name: Destroy ephemeral security groups (if needed)
      ec2_group:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}"
        name: "{{ item.security_group_name }}"
        state: absent
      vars:
        vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}"
      loop: "{{ platforms }}"
      loop_control:
        index_var: index
        label: "{{ item.name }}"
      when: item.security_groups | length == 0

    - name: Destroy ephemeral keys (if needed)
      ec2_key:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        name: "{{ item.key_name }}"
        state: absent
      loop: "{{ platforms }}"
      loop_control:
        index_var: index
        label: "{{ item.name }}"
      when: item.key_inject_method == "ec2"

ok: [localhost] => {
    "instance_config": {}
}

Expected Results

when I run a destroy either nothing happens or

Actual Results

PLAY [Destroy] *****************************************************************

TASK [Validate platform configurations] ****************************************
ok: [localhost] => (item=certificate-configure)
ok: [localhost] => (item=certificate-configure-arm)

TASK [Look up subnets to determine VPCs (if needed)] ***************************
ok: [localhost] => (item=certificate-configure)
ok: [localhost] => (item=certificate-configure-arm)

TASK [Validate discovered information] *****************************************
ok: [localhost] => (item=certificate-configure)
ok: [localhost] => (item=certificate-configure-arm)

TASK [Destroy ephemeral EC2 instances] *****************************************
changed: [localhost] => (item=certificate-configure)
ok: [localhost] => (item=certificate-configure-arm)

TASK [you dead yet] ************************************************************
fatal: [localhost]: FAILED! => {"msg": "The task includes an option with an undefined variable. The error was: {{ ec2_instances_async.results[0].instances[index] }}: list object has no element 1. list object has no element 1. {{ ec2_instances_async.results[0].instances[index] }}: list object has no element 1. list object has no element 1\n\nThe error appears to be in '/Users/matt.horwood/git/dev-ops-ansible/roles/certificate-configure/molecule/default/destroy.yml': line 96, column 7, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line appears to be:\n\n\n    - name: you dead yet\n      ^ here\n"}
ok: [localhost] => (item=certificate-configure)

Code of Conduct

• I agree to follow the Ansible Code of Conduct

[gravesm]

@matt-horwood-mayden Can you clarify what the problem is that you are having with the amazon.aws collection? You are passing a list of instance-ids to the ec2_instance module. Are you saying this task is deleting more than those instance ids? Can you provide the logs with verbose output enabled?

[matt-horwood-mayden]

Hi @gravesm

We have been using molecule testing to re-write all our roles with Ansible 9, that all works great and looks to work as we want.

But we had an instance created outside of Ansible and had no tags or name that would clash with Ansible molecule testing, but after someone had run molecule test that random instance was gone.

So I did a test and made an instance from the webUI, then ran molecule test and after the destroy task of molecule testing my instance was gone too.

It seems that if you hand the ec2_instance module an empty instance_ids it will destroy all instance in that account

[gravesm]

OK, thanks, that clarifies things. The module does delete everything if the only filter you give it is an empty list of instance ids. In this case, the module isn't clear exactly what the intended behavior should be, but I think this is a bug as it shouldn't be so easy to do so.

[matt-horwood-mayden]

great, will work on some code to stop the list being empty or skip it

[matt-horwood-mayden]

Here is the updated playbook, this will skip the Destroy ephemeral EC2 instances task if the list is empty

---
- name: Destroy
  hosts: localhost
  connection: local
  gather_facts: false
  no_log: "{{ molecule_no_log }}"
  collections:
    - community.aws
  vars:
    # Run config handling
    default_run_id: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}"
    default_run_config:
      run_id: "{{ default_run_id }}"

    run_config_path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/run-config.yml"
    run_config_from_file: "{{ (lookup('file', run_config_path, errors='ignore') or '{}') | from_yaml }}"
    run_config: '{{ default_run_config | combine(run_config_from_file) }}'

    # Platform settings handling
    default_aws_profile: "{{ lookup('env', 'AWS_PROFILE') }}"
    default_key_inject_method: cloud-init # valid values: [cloud-init, ec2]
    default_key_name: "molecule-{{ run_config.run_id }}"
    default_security_group_name: "molecule-{{ run_config.run_id }}"

    platform_defaults:
      aws_profile: "{{ default_aws_profile }}"
      key_inject_method: "{{ default_key_inject_method }}"
      key_name: "{{ default_key_name }}"
      region: ""
      security_group_name: "{{ default_security_group_name }}"
      security_groups: []
      vpc_id: ""
      vpc_subnet_id: ""

    # Merging defaults into a list of dicts is, it turns out, not straightforward
    platforms: >-
      {{ [platform_defaults | dict2items]
           | product(molecule_yml.platforms | map('dict2items') | list)
           | map('flatten', levels=1)
           | list
           | map('items2dict')
           | list }}

    # Stored instance config
    instance_config: "{{ (lookup('file', molecule_instance_config, errors='ignore') or '[]') | from_yaml }}"
  pre_tasks:
    - name: Validate platform configurations
      assert:
        that:
          - platforms | length > 0
          - platform.name is string and platform.name | length > 0
          - platform.aws_profile is string
          - platform.key_inject_method is in ["cloud-init", "ec2"]
          - platform.key_name is string and platform.key_name | length > 0
          - platform.region is string
          - platform.security_group_name is string and platform.security_group_name | length > 0
          - platform.security_groups is sequence
          - platform.vpc_id is string
          - platform.vpc_subnet_id is string and platform.vpc_subnet_id | length > 0
        quiet: true
      loop: '{{ platforms }}'
      loop_control:
        loop_var: platform
        label: "{{ platform.name }}"
  tasks:
    - name: Look up subnets to determine VPCs (if needed)
      ec2_vpc_subnet_info:
        subnet_ids: "{{ item.vpc_subnet_id }}"
      loop: "{{ platforms }}"
      loop_control:
        label: "{{ item.name }}"
      when: not item.vpc_id
      register: subnet_info

    - name: Validate discovered information
      assert:
        that: platform.vpc_id or (subnet_info.results[index].subnets | length > 0)
        quiet: true
      loop: "{{ platforms }}"
      loop_control:
        loop_var: platform
        index_var: index
        label: "{{ platform.name }}"

    - name: Destroy ephemeral EC2 instances
      ec2_instance:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        instance_ids: "{{ item.instance_ids }}"
        state: absent
      when: item is defined
      loop: "{{ instance_config }}"
      loop_control:
        label: "{{ item.instance }}"
      register: ec2_instances_async

    - name: you dead yet
      ec2_instance_info:
        instance_ids: "{{ instance.instance_id }}"
      vars:
        instance: "{{ ec2_instances_async.results[index].instances[0] }}"
      loop: "{{ ec2_instances_async.results }}"
      loop_control:
        index_var: index
        label: "{{ platforms[index].name }}"
      until: instance.state.name == "terminated"
      when: ec2_instances_async is changed
      retries: 300

    - name: Write Molecule instance configs
      copy:
        dest: "{{ molecule_instance_config }}"
        content: "{{ [] | to_yaml }}"

    - name: Destroy ephemeral security groups (if needed)
      ec2_group:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        vpc_id: "{{ item.vpc_id or vpc_subnet.vpc_id }}"
        name: "{{ item.security_group_name }}"
        state: absent
      vars:
        vpc_subnet: "{{ subnet_info.results[index].subnets[0] }}"
      loop: "{{ platforms }}"
      loop_control:
        index_var: index
        label: "{{ item.name }}"
      when: item.security_groups | length == 0

    - name: Destroy ephemeral keys (if needed)
      ec2_key:
        profile: "{{ item.aws_profile | default(omit) }}"
        region: "{{ item.region | default(omit) }}"
        name: "{{ item.key_name }}"
        state: absent
      loop: "{{ platforms }}"
      loop_control:
        index_var: index
        label: "{{ item.name }}"
      when: item.key_inject_method == "ec2"