Model Demo: Detectors Zoo - MMDetection

NOTE: The Jupyter Notebook below is included in the Chimera SDK and can be run interactively by running the following CLI command:

$ quadric sdk notebook

From the Jupyter Notebook window in your browser, select the notebook named /quadric/sdk-cli/examples/models/zoo/detectors_zoo/mmdetection.ipynb.

MMDetection

MMDetection is an open source object detection toolbox based on PyTorch. It is a part of the OpenMMLab project.

Please refer the MMDetection github for more details.

This notebook experiments MMDetection with MMDeploy as a tool to export ONNX.

Environment Setup

!pip3 install -r ../../../requirements.txt -q
!mim install -r ../../../requirements_mim.txt -q

import gc
import os
import pickle
import sys
from pathlib import Path
import shutil
import zipfile

import numpy as np
import onnx
from onnxruntime import InferenceSession
import torch
from PIL import Image

from examples.models.zoo.zoo_utils import (
    download_file,
    get_mmdeploy_parameters,
    get_transforms_and_dataset,
    install_openmmlab_github,
    onnx_check_and_simplify,
    split_onnx,
)
import examples.models.zoo.fix_registry
from tvm.contrib.epu.chimera_job.chimera_job import ChimeraJob
import tvm.contrib.epu.chimera_job.constants as sdk_constants
from sdk_cli.lib.inference import InferenceEngine, batch_inference
from sdk_cli.lib.quantize import QuantizedONNXModel, quantize_onnx_model
from sdk_cli.utils.datasets.COCO import COCO80CLASSES
from sdk_cli.utils.models.mmdeploy import MMDeployVariant
from sdk_cli.utils.models.mmdet import MMDetModelVariant
from sdk_cli.utils.networks import ChimeraNetwork
from sdk_cli.visualizers.layouter import (
    DetectorsLayouter,
    VisualizeHandle,
    VisualizeObject,
    get_visualize_handle,
)
from examples.models.zoo.detectors_zoo.detectors_utils import (
    get_mmdet_parameters,
    mmdeploy_detectors_postprocessing,
)

CUR_DIR = os.path.abspath(os.getcwd())

Install MMDtection and MMDeploy

github_names = [
    ("mmdetection", "3.2.0", None),
    ("mmdeploy", "1.3.1", None),
]
symlink_list = [
    ("configs", "mmdetection"),
    ("mmdet", "mmdetection"),
    ("model-index.yml", "mmdetection"),
]
install_openmmlab_github(github_names, symlink_list)

Select Model

This notebook can experiment the following models.

• atss_r50
• autoassign_r50
• boxinst_r50_fpn
• centernet_update_r50
• condinst_r50_fpn
• conditional_detr_r50
• dab_detr_r50
• decoupled_solo_r50
• decoupled_solo_light_r50
• detr_r50
• dynamic_rcnn_r50_fpn
• faster_rcnn_r101_fpn
• faster_rcnn_r50_fpn
• faster_rcnn_r50_pafpn
• faster_rcnn_regnetx_1_6gf_fpn_ms
• faster_rcnn_regnetx_3_2gf_fpn
• faster_rcnn_regnetx_3_2gf_fpn_ms
• faster_rcnn_regnetx_400mf_fpn_ms
• faster_rcnn_regnetx_800mf_fpn_ms
• fcos_r101_caffe_fpn_gn_head
• fcos_r50_caffe_fpn_gn_head
• fovea_r50
• freeanchor_r50
• fsaf_r50
• gfl_r50
• lad_r50
• ld_r101_gflv1_r101_dcn_fpn
• ld_r18_gflv1_r101_fpn
• ld_r34_gflv1_r101_fpn
• ld_r50_gflv1_r101_fpn
• mask_rcnn_r101_fpn
• mask_rcnn_r50_fpn
• mask_rcnn_regnetx_1_6gf_fpn_ms_poly
• mask_rcnn_regnetx_3_2gf_fpn
• mask_rcnn_regnetx_3_2gf_fpn_ms
• mask_rcnn_regnetx_400mf_fpn_ms_poly
• mask_rcnn_regnetx_800mf_fpn_ms_poly
• paa_r50
• pisa_retinanet_r50
• retinanet_r50
• retinanet_r50_fpn_openimages
• retinanet_regnetx_1_6gf_fpn
• retinanet_regnetx_3_2gf_fpn
• retinanet_regnetx_800mf_fpn
• solo_r50
• ssdlite_mobilenetv2
• yolof_r50
• yolov3_d53_320x320
• yolov3_d53_608x608
• yolov3_mobilenetv2
• yolox_s
• yolox_tiny

Now we are going to experiment ssdlite_mobilenetv2 as an example.

## Select target model.
MODEL_NAME = MMDetModelVariant.ssdlite_mobilenetv2

mmdet_parameters = get_mmdet_parameters(MODEL_NAME)

model_input_size = mmdet_parameters.model_input_size
mmdeploy_variant = MMDeployVariant.mmdetection

mmdeploy_parameters = get_mmdeploy_parameters(mmdeploy_variant, mmdet_parameters)

Load Pretrained Model

weights = Path(mmdeploy_parameters.weights_url).name

if not Path(weights).exists():
    download_file(mmdeploy_parameters.weights_url, weights)
    print(f"{weights} is downloaded.")

Export Torch Model

os.chdir(Path(CUR_DIR) / "mmdeploy")

from mmdeploy.apis import torch2onnx
import torch

os.chdir(CUR_DIR)

from mmdeploy.utils import load_config
from mmdeploy.apis import build_task_processor

## load deploy_config
deploy_config, model_config = load_config(
    mmdeploy_parameters.deploy_config, mmdeploy_parameters.config
)

## create model an inputs
task_processor = build_task_processor(model_config, deploy_config, "cpu")
torch_model = task_processor.build_pytorch_model(weights)

Export ONNX

onnx_file = f"{MODEL_NAME}.onnx"

input_edges = deploy_config.onnx_config.input_names
output_edges = deploy_config.onnx_config.output_names

if MODEL_NAME in [
    MMDetModelVariant.decoupled_solo_r50,
    MMDetModelVariant.decoupled_solo_light_r50,
    MMDetModelVariant.solo_r50,
]:
    input_tensor = torch.randn(1, 3, *model_input_size[::-1], requires_grad=False)

    # Export the model
    torch.onnx.export(
        torch_model,  # model being run
        input_tensor,  # model input (or a tuple for multiple inputs)
        onnx_file,  # where to save the model (can be a file or file-like object)
        export_params=True,  # store the trained parameter weights inside the model file
        do_constant_folding=True,  # whether to execute constant folding for optimization
        opset_version=sdk_constants.DEFAULT_ONNX_OPSET,
        input_names=input_edges,  # the model's input names
        output_names=output_edges,  # the model's output names
    )
else:
    sample_image = "../../../common/calibration/face/daniel_maverick.png"
    Image.open(sample_image).resize(mmdet_parameters.model_input_size).save("image.jpg")

    # Convert the model to ONNX
    torch2onnx(
        "image.jpg",
        work_dir=".",
        save_file=onnx_file,
        deploy_cfg=mmdeploy_parameters.deploy_config,
        model_cfg=mmdeploy_parameters.config,
        model_checkpoint=weights,
        device="cpu",
    )

onnx_model = onnx_check_and_simplify(onnx.load(onnx_file), mmdet_parameters.skip_shape_inference)
onnx.save(onnx_model, onnx_file)

print(f"ONNX is exported and simplified as {onnx_file}")

gc.collect();

Split ONNX

target_onnx_file, second_onnx_file = split_onnx(
    onnx_file,
    mmdet_parameters,
    input_edges=input_edges,
    output_edges=output_edges,
)

if mmdet_parameters.model_head:
    # We will use torch_model for this head postprocessing.
    # Set None to second_onnx_file for later inference demo.
    second_onnx_file = None

gc.collect();

Quantization

transforms, coco_like_subset_of_dataset = get_transforms_and_dataset(mmdeploy_parameters)

quantized_onnx_model = quantize_onnx_model(
    target_onnx_file,
    coco_like_subset_of_dataset,
    asymmetric_activation=True,
)
print(
    f"quantized onnx: {quantized_onnx_model.model_path}, tranges file: {quantized_onnx_model.tensor_ranges_path}"
)

gc.collect();

Compilation

cgc_job = ChimeraJob(
    model_p=str(quantized_onnx_model.model_path),
    trange_file=str(quantized_onnx_model.tensor_ranges_path),
)
cgc_job.compile(quiet=True)
print(cgc_job)

Demo

Inference

all_image_paths = [
    "../../../common/calibration/face/daniel_maverick.png",
    "../../../common/calibration/face/sales_squad_jani.jpg",
    "../../../common/calibration/coco-like/33823288584_1d21cf0a26_k.jpeg",
]

all_images = []
for image_path in all_image_paths:
    image = Image.open(image_path)
    transformed_image = transforms(image)
    all_images.append(np.expand_dims(transformed_image.numpy(), axis=0).astype(np.float32))

engines = {
    InferenceEngine.CHIMERA_ORT_INT8: cgc_job,
    InferenceEngine.CHIMERA_ISS_INT8: cgc_job,
}

head_session = (
    None
    if second_onnx_file is None
    else InferenceSession(onnx.load(second_onnx_file).SerializeToString())
)

outputs_per_inference_engine = {}
THREADS = min(len(all_images), 6)
for inference_engine, engine in engines.items():
    all_backbone_outputs = batch_inference(
        inference_engine,
        engine,
        all_images,
        head_session=head_session,
        threads=THREADS,
    )
    outputs_per_inference_engine[inference_engine] = all_backbone_outputs

Run Statistics

print(cgc_job)
cgc_job.plot_run_statistics()

Postprocessing

## Get visualize handle for visualization.
visualize_handle = get_visualize_handle(
    mmdet_parameters.visualize_object, mmdet_parameters.visual_threshold
)

## Retrieve inference results per engine.
detections_per_inference_engine = {}

for inference_engine, all_outputs in outputs_per_inference_engine.items():
    detections_per_inference_engine[inference_engine] = []
    for outputs, image_file in zip(all_outputs, all_image_paths):
        if mmdet_parameters.model_head:
            outputs = mmdet_parameters.model_head(
                outputs, image_file, model_input_size, torch_model
            )
        detections = mmdeploy_detectors_postprocessing(
            outputs,
            Image.open(image_file).size,
            model_input_size,
            nms=mmdet_parameters.nms,
            score_threshold=mmdet_parameters.visual_threshold,
            nms_threshold=mmdet_parameters.nms_threshold,
        )
        detections_per_inference_engine[inference_engine].append(detections)

Display Inference Results

%matplotlib inline

for i in range(len(all_images)):
    classes = mmdet_parameters.classes
    layouter = DetectorsLayouter(
        visualize_handle,
        image=all_image_paths[i],
        classes=classes if len(classes) else COCO80CLASSES,
        show_class=mmdet_parameters.show_class,
    )
    for inference_engine, all_outputs in detections_per_inference_engine.items():
        layouter.add_data(
            bboxes=all_outputs[i][0],
            masks=all_outputs[i][1],
            title=f"{MODEL_NAME}:\n{str(inference_engine).upper()}",
        )

    layouter.display()

Citation

@article{mmdetection,
  title   = {{MMDetection}: Open MMLab Detection Toolbox and Benchmark},
  author  = {Chen, Kai and Wang, Jiaqi and Pang, Jiangmiao and Cao, Yuhang and
             Xiong, Yu and Li, Xiaoxiao and Sun, Shuyang and Feng, Wansen and
             Liu, Ziwei and Xu, Jiarui and Zhang, Zheng and Cheng, Dazhi and
             Zhu, Chenchen and Cheng, Tianheng and Zhao, Qijie and Li, Buyu and
             Lu, Xin and Zhu, Rui and Wu, Yue and Dai, Jifeng and Wang, Jingdong
             and Shi, Jianping and Ouyang, Wanli and Loy, Chen Change and Lin, Dahua},
  journal= {arXiv preprint arXiv:1906.07155},
  year={2019}
}

@misc{=mmdeploy,
    title={OpenMMLab's Model Deployment Toolbox.},
    author={MMDeploy Contributors},
    howpublished = {\url{https://github.com/open-mmlab/mmdeploy}},
    year={2021}
}