Make molecule reuse prerun from ansible-lint (#3077)

As ansible-lint is able to detect repository layout and run prepare
ansible from running it, we rely on its logic in order to assure
ansible is able to run properly.

This means that dependencies will be installed into .cache folder
and that the role import path will be updated to include it.

.config/molecule/config.yml

@@ -1 +1,2 @@
 # This is loaded by default
+prerun: true

.pre-commit-config.yaml

@@ -54,15 +54,16 @@ repos:
         entry: mypy src/
         pass_filenames: false
         additional_dependencies:
+          - ansible-lint>=5.0.5
           - packaging
           - enrich>=1.2.5
           - subprocess-tee>=0.2.0
   - repo: https://github.com/pre-commit/mirrors-pylint
-    rev: v2.6.0
+    rev: v2.7.2
     hooks:
       - id: pylint
         additional_dependencies:
-          - ansible-base
+          - ansible-lint>=5.0.5
           - enrich>=1.2.5
           - subprocess-tee>=0.2.0
           - testinfra

.readthedocs.yml

@@ -12,4 +12,5 @@ python:
     - method: pip
       path: .
       extra_requirements:
+        - ansible-base
         - docs

docs/configuration.rst

@@ -6,6 +6,27 @@ Configuration
 
 .. _variable substitution:
 
+
+Prerun
+------
+
+In order to help Ansible find used modules and roles, molecule will perform
+a prerun set of actions. These involve installing dependencies from
+``requirements.yml`` specified at project level, install a standalone role
+or a collection. The destination is ``project_dir/.cache`` and the code itself
+is reused from ansible-lint, which has to do the same actions.
+
+This assures that when you include a role inside molecule playbooks, Ansible
+will be able to find that role, and that the include is exactly the same as
+the one you are expecting to use in production.
+
+If for some reason the prerun action does not suits your needs, you can still
+disable it by adding `prerun: false` inside the configuration file.
+
+Keep in mind that you can add this value to ``.config/molecule/config.yml``
+file in order to avoid adding it to each scenario.
+
+
 Variable Substitution
 ---------------------
 

mypy.ini

@@ -7,6 +7,9 @@ warn_redundant_casts = True
 no_implicit_optional = True
 
 # 3rd party ignores, to remove once they add hints
+[mypy-ansiblelint.*]
+ignore_missing_imports = True
+
 [mypy-cerberus.*]
 ignore_missing_imports = True
 

setup.cfg

@@ -66,6 +66,7 @@ setup_requires =
 
 # These are required in actual runtime:
 install_requires =
+    ansible-lint >= 5.0.5  # only for the prerun functionality
     cerberus >= 1.3.1
     click >= 7.0, < 8.0  # https://github.com/click-contrib/click-help-colors/issues/12
     click-completion >= 0.5.1
@@ -120,9 +121,10 @@ test =
     pytest-xdist >= 2.1.0
     pytest >= 6.1.2
 lint =
-    ansible-lint[core,yamllint] >= 5.0.2, < 6
+    # ansible-lint is now a core dependency, duplicating it here would confuse pip
     flake8 >= 3.8.4
     pre-commit >= 2.10.1
+    yamllint
 
 [options.entry_points]
 console_scripts =

src/molecule/command/base.py

@@ -28,6 +28,7 @@
 from typing import Any, Callable
 
 import click
+from ansiblelint.prerun import prepare_environment
 from click_help_colors import HelpColorsCommand, HelpColorsGroup
 
 import molecule.scenarios
@@ -105,6 +106,11 @@ def execute_cmdline_scenarios(scenario_name, args, command_args, ansible_args=()
         )
 
     for scenario in scenarios:
+
+        if scenario.config.config["prerun"]:
+            LOG.info("Performing prerun...")
+            prepare_environment()
+
         if command_args.get("subcommand") == "reset":
             LOG.info("Removing %s" % scenario.ephemeral_directory)
             shutil.rmtree(scenario.ephemeral_directory)

src/molecule/config.py

@@ -348,6 +348,7 @@ def _get_defaults(self) -> MutableMapping:
                 "safe_files": [],
             },
             "platforms": [],
+            "prerun": True,
             "provisioner": {
                 "name": "ansible",
                 "config_options": {},

src/molecule/data/validate-dockerfile.yml

@@ -1,6 +1,8 @@
 #!/usr/bin/env ansible-playbook
 ---
 - hosts: localhost
+  collections:
+    - community.docker
   vars:
     platforms:
       # platforms supported as being managed by molecule/ansible, this does

src/molecule/provisioner/ansible.py

@@ -453,6 +453,7 @@ def default_env(self):
                         ),
                         "/usr/share/ansible/roles",
                         "/etc/ansible/roles",
+                        *os.environ.get("ANSIBLE_ROLES_PATH", "").split(":"),
                     ]
                 ),
                 self._config.ansible_collections_path: ":".join(collections_path_list),

src/molecule/test/unit/provisioner/test_ansible.py

@@ -202,7 +202,11 @@ def test_env_property(_instance):
     "config_instance", ["_provisioner_section_data"], indirect=True
 )
 def test_env_appends_env_property(_instance):
-    x = [
+
+    # molecule could decide to add extra paths, so we only want to check
+    # that those that we need are kept inside the list
+    roles_path_list = _instance.env["ANSIBLE_ROLES_PATH"].split(":")
+    for x in [
         util.abs_path(
             os.path.join(_instance._config.scenario.ephemeral_directory, "roles")
         ),
@@ -213,8 +217,8 @@ def test_env_appends_env_property(_instance):
         "/usr/share/ansible/roles",
         "/etc/ansible/roles",
         util.abs_path(os.path.join(_instance._config.scenario.directory, "foo", "bar")),
-    ]
-    assert x == _instance.env["ANSIBLE_ROLES_PATH"].split(":")
+    ]:
+        assert x in roles_path_list
 
     x = _instance._get_modules_directories()
     x.append(

tox.ini

@@ -35,6 +35,7 @@ passenv =
 setenv =
     ANSIBLE_CONFIG={toxinidir}/.ansible.cfg
     ANSIBLE_DISPLAY_FAILED_STDERR=1
+    ANSIBLE_NOCOWS=1
     ANSIBLE_VERBOSITY=1
     MOLECULE_NO_LOG=0
     PYTHONDONTWRITEBYTECODE=1
@@ -44,9 +45,10 @@ deps =
     devel: ansible>=2.10.0a2,<2.11
     # pytest-molecule not used but we want to check that it does not conflict
     devel: git+https://github.com/ansible-community/pytest-molecule#egg=pytest-molecule
-    dockerfile: ansible>=2.9.12
+    dockerfile: ansible>=2.10
     selinux
     py{36,37}: importlib-metadata<2,>=0.12
+    py: ansible-base
 extras =
     docker
     lint
@@ -58,10 +60,11 @@ extras =
 ;     sh -c 'find {homedir}/.cache -type d -path "*/molecule_*" -exec rm -rfv \{\} +;'
 commands =
     ansibledevel: ansible-galaxy install git+https://github.com/ansible-collections/community.general.git
+    dockerfile: ansible-galaxy install community.docker
     # failsafe as pip may install incompatible dependencies
     pip check
     # failsafe for preventing changes that may break pytest collection
-    sh -c "PYTEST_ADDOPTS= python -m pytest -p no:cov --collect-only 2>&1 >{envlogdir}/collect.log"
+    sh -c "PYTEST_ADDOPTS= python -m pytest -p no:cov --collect-only"
     # -n auto used only on unit as is not supported by functional yet
     # html report is used by Zuul CI to display reports
     python -m pytest src/molecule/test/unit/ {env:_EXTRAS} {env:PYTEST_ADDOPTS:} {posargs}
@@ -102,6 +105,7 @@ commands =
     'import pathlib; '\
     'docs_dir = pathlib.Path(r"{toxinidir}") / "docs/docstree/html"; index_file = docs_dir / "index.html"; print(f"\nDocumentation available under `file://\{index_file\}`\n\nTo serve docs, use `python3 -m http.server --directory \{docs_dir\} 0`\n")'
 extras =
+    ansible-base
     docs
 
 [testenv:docs-livereload]