Added acceptance test framework and test suites; Updates to CI/CD pip…

…eline; Added PyTorch Uniformer, MMAction2 BMN model and W8A16 ResNet50 - Added acceptance tests framework and test suite for most pytorch and some tf models - Updates to PR CI/CD CMake and Jenkins files - Added W8A16 quantization and artifacts for pytorch ResNet50 model - Releasing next package release of aimet model zoo (installable wheel file binaries) - Added code, documentation and artifacts the following new Pytorch models : UniFormer, MMAction2 BMN Signed-off-by: Hanwen Xiong <[email protected]>
quic · Jun 19, 2023 · 7e1bea4 · 7e1bea4
1 parent 667e804
commit 7e1bea4
Show file tree

Hide file tree

Showing 187 changed files with 5,145 additions and 569 deletions.
diff --git a/NightlyTests/CMakeLists.txt → AcceptanceTests/CMakeLists.txt b/NightlyTests/CMakeLists.txt → AcceptanceTests/CMakeLists.txt
diff --git a/AcceptanceTests/tensorflow/conftest.py b/AcceptanceTests/tensorflow/conftest.py
@@ -0,0 +1,55 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+"""pytest configuration fixtures"""
+
+import pytest
+import os
+import warnings
+from pathlib import Path
+
+
+@pytest.fixture(scope='module')
+def test_data_path():
+    """
+    This fixture will return the path to testing data for all models. When no
+    DEPENDENCY_DATA_PATH is detected, None is returned and all acceptance tests
+    which depend on testing data will be xfailed
+    """
+    try:
+        data_path = Path(os.getenv('DEPENDENCY_DATA_PATH'))
+    except TypeError:
+        warnings.warn('In order to successfully proceed with acceptance test, please set DEPENDENCY_DATA_PATH')
+        data_path = None
+
+    yield data_path
+
+@pytest.fixture(scope='module')
+def tiny_imageNet_root_path(test_data_path):
+    if test_data_path is not None:
+        tiny_imageNet_root_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced').as_posix() 
+    else:
+        tiny_imageNet_root_path = None
+
+    yield tiny_imageNet_root_path
+
+
+@pytest.fixture(scope='module')
+def tiny_mscoco_tfrecords(test_data_path):
+    if test_data_path is not None:
+        tiny_mscoco_tfrecords = (test_data_path / "model_zoo_datasets/tiny_mscoco_tfrecords").as_posix() 
+    else:
+        tiny_mscoco_tfrecords = None
+
+    yield tiny_mscoco_tfrecords
+
+
diff --git a/AcceptanceTests/tensorflow/pytest.ini b/AcceptanceTests/tensorflow/pytest.ini
@@ -0,0 +1,21 @@
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+# content of pytest.ini
+[pytest]
+markers =
+    cuda: test that require CUDA to be installed
+    image_classification : test that belong to task of image classifcation 
+    slow: test that runs slow 
+    nlp: tests that belong to natual language process task
+    object_detection: tests that belong to object detection task
+    pose_estimation: tests that belong to pose estimation task
+    sementic_segmentation: tests that belong to sementic segmentation task
+    super_resolution: tests that belong to super resolution task  
+    slow: tests that runs longer than 1 minutes per model config 
diff --git a/AcceptanceTests/tensorflow/test_mobiledet_edgetpu_quanteval.py b/AcceptanceTests/tensorflow/test_mobiledet_edgetpu_quanteval.py
@@ -0,0 +1,41 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for mobiledet edgetpu"""
+
+import pytest
+from aimet_zoo_tensorflow.mobiledetedgetpu.evaluators import mobiledet_edgetpu_quanteval
+
+@pytest.mark.slow 
+@pytest.mark.cuda 
+@pytest.mark.object_detection
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["mobiledet_w8a8"])
+def test_quanteval_mobiledet_edgetpu(model_config, tiny_mscoco_tfrecords):
+    """mobiledet edgetpu image classification test"""
+
+    if tiny_mscoco_tfrecords is None:
+        pytest.fail(f'Dataset path is not set')
+
+    mobiledet_edgetpu_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            tiny_mscoco_tfrecords,
+            "--annotation-json-file",
+            tiny_mscoco_tfrecords+"/instances_val2017.json"
+        ]
+    )
+
+
+
diff --git a/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py b/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py
@@ -0,0 +1,38 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for mobilenet_v2_tf2_quanteval image classification"""
+
+import pytest
+from aimet_zoo_tensorflow.mobilenet_v2_tf2.evaluators import mobilenet_v2_tf2_quanteval
+
+@pytest.mark.cuda 
+@pytest.mark.object_detection
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["resnet50_w8a8"])
+def test_quanteval_mobilenet_v2(model_config, tiny_imageNet_root_path):
+    """mobilenet_v2_tf2 image classification acceptance test"""
+
+    if tiny_imageNet_root_path is None:
+        pytest.fail(f'Dataset path is not set')
+
+    mobilenet_v2_tf2_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            tiny_imageNet_root_path,
+        ]
+    )
+
+
+
diff --git a/AcceptanceTests/tensorflow/test_resnet50_tf2_quanteval.py b/AcceptanceTests/tensorflow/test_resnet50_tf2_quanteval.py
@@ -0,0 +1,38 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for resnet50_tf2_quanteval image classification"""
+
+import pytest
+from aimet_zoo_tensorflow.resnet50_tf2.evaluators import resnet50_tf2_quanteval
+
+@pytest.mark.cuda 
+@pytest.mark.object_detection
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["resnet50_w8a8"])
+def test_quanteval_ssd_mobilenetv2(model_config, tiny_imageNet_root_path):
+    """resnet50_tf2 image classification acceptance test"""
+
+    if tiny_imageNet_root_path is None:
+        pytest.fail(f'Dataset path is not set')
+
+    resnet50_tf2_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            tiny_imageNet_root_path,
+        ]
+    )
+
+
+
diff --git a/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py b/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py
@@ -0,0 +1,41 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for ssd_mobilenetv2_quanteval edgetpu"""
+
+import pytest
+from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenetv2_quanteval
+
+@pytest.mark.slow 
+@pytest.mark.cuda 
+@pytest.mark.object_detection
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["ssd_mobilenetv2_w8a8"])
+def test_quanteval_ssd_mobilenetv2(model_config, tiny_mscoco_tfrecords):
+    """ssd mobilenetv2 object detection acceptance test"""
+
+    if tiny_mscoco_tfrecords is None:
+        pytest.fail(f'Dataset path is not set')
+
+    ssd_mobilenetv2_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            tiny_mscoco_tfrecords,
+            "--annotation-json-file",
+            tiny_mscoco_tfrecords+"/instances_val2017.json"
+        ]
+    )
+
+
+
diff --git a/NightlyTests/torch/CMakeLists.txt → AcceptanceTests/torch/CMakeLists.txt b/NightlyTests/torch/CMakeLists.txt → AcceptanceTests/torch/CMakeLists.txt
@@ -13,22 +13,19 @@ else (ENABLE_CUDA)
     set(USE_CUDA False)
 endif (ENABLE_CUDA)
 
-#add_custom_target(AcceptanceTests.TorchDependencies
-#      COMMAND ${CMAKE_COMMAND} -E env
-#      "${AIMET_PYTHONPATH}"
-#      "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/dependencies.py ${CMAKE_CURRENT_SOURCE_DIR}/resnet18_eval_scores.csv
-#       ${USE_CUDA})
-
 add_custom_target( AcceptanceTests.Torch )
 
-file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py")
+file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_resnet_quanteval.py")
     foreach(filename ${files})
+
         get_filename_component( testname "${filename}" NAME_WE )
+        message(STATUS "Testname: " testname)
+
         add_custom_target(AcceptanceTests.Torch.${testname}
             VERBATIM COMMAND ${CMAKE_COMMAND} -E env
-             "${AIMET_PYTHONPATH}"
-             ${Python3_EXECUTABLE} -m pytest -s ${filename} ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml)
+                             ${MZ_PYTHONPATH} 
+                             ${Python3_EXECUTABLE} -m pytest -s ${filename} 
+                             ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) 
 
-        add_dependencies(  AcceptanceTests.Torch.${testname} AcceptanceTests.TorchDependencies )
         add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} )
-    endforeach( filename )
+    endforeach( filename )
diff --git a/AcceptanceTests/torch/conftest.py b/AcceptanceTests/torch/conftest.py
@@ -0,0 +1,102 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+"""pytest configuration fixtures"""
+
+import pytest
+import os
+import warnings
+from pathlib import Path
+
+
+@pytest.fixture(scope='module')
+def test_data_path():
+    """
+    This fixture will return the path to testing data for all models. When no
+    DEPENDENCY_DATA_PATH is detected, None is returned and all acceptance tests
+    which depend on testing data will be xfailed
+    """
+    try:
+        data_path = Path(os.getenv('DEPENDENCY_DATA_PATH'))
+    except TypeError:
+        warnings.warn('In order to successfully proceed with acceptance test, please set DEPENDENCY_DATA_PATH')
+        data_path = None
+
+    yield data_path
+
+@pytest.fixture(scope='module')
+def tiny_imageNet_root_path(test_data_path):
+    if test_data_path is not None:
+        tiny_imageNet_root_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced').as_posix() 
+    else:
+        tiny_imageNet_root_path = None
+
+    yield tiny_imageNet_root_path
+
+
+@pytest.fixture(scope='module')
+def tiny_imageNet_validation_path(test_data_path):
+    if test_data_path is not None:
+        tiny_imageNet_validation_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced/val').as_posix() 
+    else:
+        tiny_imageNet_validation_path = None
+
+    yield tiny_imageNet_validation_path
+
+@pytest.fixture(scope='module')
+def tiny_imageNet_train_path(test_data_path):
+    if test_data_path is not None:
+        tiny_imageNet_train_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced/train').as_posix() 
+    else:
+        tiny_imageNet_train_path = None
+
+    yield tiny_imageNet_train_path
+
+
+@pytest.fixture(scope='module')
+def tiny_mscoco_validation_path(test_data_path):
+    if test_data_path is not None:
+        tiny_mscoco_validation_path = (test_data_path / "model_zoo_datasets/tiny_coco/val_2017").as_posix() 
+    else:
+        tiny_mscoco_validation_path = None
+
+    yield tiny_mscoco_validation_path.as_posix()
+
+
+@pytest.fixture(scope='module')
+def tiny_cityscapes_path(test_data_path):
+    if test_data_path is not None:
+        tiny_cityscapes_path = (test_data_path / "model_zoo_datasets/tiny_cityscapes").as_posix() 
+    else:
+        tiny_cityscapes_path = None
+
+    yield tiny_cityscapes_path
+
+
+@pytest.fixture(scope='module')
+def super_resolution_set5_path(test_data_path):
+    if test_data_path is not None:
+        super_resolution_set5_path = (test_data_path / "model_zoo_datasets/super_resolution_data/Set5/image_SRF_4_HR").as_posix() 
+    else:
+        super_resolution_set5_path = None
+
+    yield super_resolution_set5_path
+
+
+@pytest.fixture(scope='module')
+def PascalVOC_segmentation_test_data_path(test_data_path):
+    if test_data_path is not None:
+        pascalVOC_segmentation_path = (test_data_path / 'PascalVOCSegmentation').as_posix() 
+    else:
+        pascalVOC_segmentation_path = None
+
+    yield pascalVOC_segmentation_path
diff --git a/NightlyTests/torch/hrnet.yaml → AcceptanceTests/torch/hrnet.yaml b/NightlyTests/torch/hrnet.yaml → AcceptanceTests/torch/hrnet.yaml
diff --git a/AcceptanceTests/torch/pytest.ini b/AcceptanceTests/torch/pytest.ini
@@ -0,0 +1,19 @@
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+[pytest]
+markers =
+    cuda: test that require CUDA to be installed
+    image_classification : test that belong to task of image classifcation 
+    slow: test that runs slow 
+    nlp: tests that belong to natual language process task
+    object_detection: tests that belong to object detection task
+    pose_estimation: tests that belong to pose estimation task
+    sementic_segmentation: tests that belong to sementic segmentation task
+    super_resolution: tests that belong to super resolution task