From 48192949c74d74bae56cca51029f1cb33cb074e6 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Tue, 7 Apr 2020 15:55:56 +0100 Subject: [PATCH] Mark some functional tests as extensive Adds 'extensive' marker to some tests and configure tox to skip them by default. The extensive tests are supposed to be indirectly covered by the full test commands and this should decrease the CI testing considerably. We do not remove these tests in order to make it easy to revive them if needed. --- molecule/test/functional/test_command.py | 11 +++++++++++ setup.cfg | 2 ++ tox.ini | 2 +- zuul.d/layout.yaml | 14 +++++++------- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/molecule/test/functional/test_command.py b/molecule/test/functional/test_command.py index e0759025a..c4d71f7ed 100644 --- a/molecule/test/functional/test_command.py +++ b/molecule/test/functional/test_command.py @@ -43,6 +43,7 @@ def driver_name(request): return request.param +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -58,6 +59,7 @@ def test_command_check(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -73,6 +75,7 @@ def test_command_cleanup(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -88,6 +91,7 @@ def test_command_converge(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -167,6 +171,7 @@ def test_command_dependency_shell( assert os.path.isdir(dependency_role) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -182,6 +187,7 @@ def test_command_destroy(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -209,6 +215,7 @@ def test_command_init_scenario(temp_dir, driver_name, skip_test): pytest.helpers.init_scenario(temp_dir, driver_name) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -325,6 +332,7 @@ def test_command_login(scenario_to_test, with_scenario, login_args, scenario_nam pytest.helpers.login(login_args, scenario_name) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -344,6 +352,7 @@ def test_command_prepare(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -359,6 +368,7 @@ def test_command_side_effect(scenario_to_test, with_scenario, scenario_name): pytest.helpers.run_command(cmd) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ @@ -388,6 +398,7 @@ def test_command_test(scenario_to_test, with_scenario, scenario_name, driver_nam pytest.helpers.test(driver_name, scenario_name) +@pytest.mark.extensive @pytest.mark.parametrize( "scenario_to_test, driver_name, scenario_name", [ diff --git a/setup.cfg b/setup.cfg index 414e28a9e..b20107af9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -150,6 +150,8 @@ filterwarnings = ignore::DeprecationWarning:cookiecutter # remove once https://github.com/pytest-dev/pytest-cov/issues/327 is released ignore::pytest.PytestWarning:pytest_cov +markers = + extensive: marks tests that we want to skip by default, as they are indirectly covered by other tests [flake8] # do not add excludes for files in repo diff --git a/tox.ini b/tox.ini index f6b1a2733..497392847 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ setenv = # -n auto used only on unit as is not supported by functional yet # html report is used by Zuul CI to display reports unit: PYTEST_ADDOPTS=molecule/test/unit/ {env:_EXTRAS} {env:PYTEST_ADDOPTS:-n auto} - functional: PYTEST_ADDOPTS=molecule/test/functional/ {env:_EXTRAS} {env:PYTEST_ADDOPTS:} + functional: PYTEST_ADDOPTS=molecule/test/functional/ {env:_EXTRAS} {env:PYTEST_ADDOPTS:-k "not extensive"} deps = ansible28: ansible>=2.8,<2.9 ansible29: git+https://github.com/ansible/ansible.git@stable-2.9 diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index 1060ece73..a04d5bcda 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -65,7 +65,7 @@ vars: tox_envlist: py36-ansible28-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - job: name: molecule-tox-py36-ansible29-unit @@ -82,7 +82,7 @@ vars: tox_envlist: py36-ansible29-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - job: name: molecule-tox-py37-ansible28-unit @@ -99,7 +99,7 @@ vars: tox_envlist: py37-ansible28-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - job: name: molecule-tox-py37-ansible29-unit @@ -116,7 +116,7 @@ vars: tox_envlist: py37-ansible29-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - job: name: molecule-tox-devel-unit @@ -137,7 +137,7 @@ vars: tox_envlist: ansibledevel-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - job: name: molecule-tox-py37-ansible28-unit @@ -150,11 +150,11 @@ - job: name: molecule-tox-py37-ansible28-functional parent: molecule-tox-py37 - timeout: 7000 + timeout: 9000 vars: tox_envlist: py37-ansible28-functional tox_environment: - PYTEST_REQPASS: 70 + PYTEST_REQPASS: 37 - project: templates: