Model Demo: Detectors3D Zoo - MMDetection3D

NOTE: The Jupyter Notebook below is included in the Chimera SDK and can be run interactively by running the following CLI command:

$ quadric sdk notebook

From the Jupyter Notebook window in your browser, select the notebook named /quadric/sdk-cli/examples/models/zoo/detectors3d_zoo/mmdetection3d.ipynb.

MMDetection3D

MMDetection3D is an open source object detection toolbox based on PyTorch, towards the next-generation platform for general 3D detection. It is a part of the OpenMMLab project.

Please refer the MMDetection3D github for more details.

Datasets

In this experiment, we use two datasets which are included in mmdetection3d repository.

• KITTI dataset
  • model example: pgd_r101_caffe_fpn_gn_kitti
• nuScenes dataset
  • model example: pointpillars_hv_fpn_sbn_nus_3d

Coordinate systems are slightly different. The following is the comparison table between those datasets.

Outputs Example

We will visualize 3d detectors outputs by mapping 3d boxes and point cloud on a camera image and a bird's eye view like the following.

Environment Setup

!pip3 install -r ../../../requirements.txt -q
!mim install -r ../../../requirements_mim.txt -q

import os
from pathlib import Path

import matplotlib.pyplot as plt
import onnx
from onnxruntime import InferenceSession
from PIL import Image
import torch

from examples.models.zoo.zoo_utils import (
    download_file,
    get_mmdeploy_parameters,
    install_openmmlab_github,
    onnx_check_and_simplify,
    split_onnx,
)
import examples.models.zoo.fix_registry
from tvm.contrib.epu.chimera_job.chimera_job import ChimeraJob
from tvm.contrib.epu.chimera_job.quantize import quadric_quantize
from tvm.contrib.epu.onnx_util import cut_onnx
from sdk_cli.lib.inference import InferenceEngine
from sdk_cli.utils.models.mmdeploy import MMDeployVariant
from sdk_cli.utils.models.mmdet3d import MMDet3DModelVariant, MMDet3DTask

from detectors3d_utils import (
    draw_projected_bboxes,
    draw_bev_image,
    KittiDataSample,
    get_imshow_extent,
    get_kitti_sample,
    get_nuscene_sample,
    get_mmdet3d_parameters,
    get_bev_image,
    ModelHelperDetector3D,
    perform_mmdet3d_inference,
)

from mmdet3d.apis.inferencers import LidarDet3DInferencer, MonoDet3DInferencer
from mmdet3d.structures import Box3DMode
import mmengine


CUR_DIR = os.path.abspath(os.getcwd())

Install OpenMMLab Repositories

github_names = [
    ("mmengine", "0.10.7", "mmengine-main"),
    ("mmdetection3d", "1.4.0", None),
    ("mmdeploy", "1.3.1", None),
]
symlink_list = [
    ("mmengine", "mmengine-main"),
    ("configs", "mmdetection3d"),
    ("demo", "mmdetection3d"),
]
install_openmmlab_github(github_names, symlink_list)

!sed -i "347s/map_location=map_location)/map_location=map_location, weights_only=False)/" mmengine/runner/checkpoint.py

Set-up MMDeploy Configs

config_text = """_base_ = ['../../_base_/onnx_config.py', '../../_base_/backends/onnxruntime.py']
codebase_config = dict(
    type='mmdet3d', task='VoxelDetection', model_type='end2end')
onnx_config = dict(
    input_names=['voxels', 'num_points', 'coors'],
    output_names=%s)
"""

output_3names = ["cls_score0", "bbox_pred0", "dir_cls_pred0"]
output_9names = output_3names + [
    "cls_score1",
    "bbox_pred1",
    "dir_cls_pred1",
    "cls_score2",
    "bbox_pred2",
    "dir_cls_pred2",
]

mmdet3d_config_path = (
    "mmdeploy/configs/mmdet3d/voxel-detection/voxel-detection_onnxruntime_static_%soutputs.py"
)

for output_num, output_names in zip([3, 9], [output_3names, output_9names]):
    config_path = mmdet3d_config_path % (output_num)
    if not Path(config_path).exists():
        with Path(config_path).open("w") as fd:
            fd.writelines(config_text % output_names)
            print(f"{config_path} is stored")

config_path = "mmdeploy/configs/mmdet3d/mono-detection/mono-detection_static.py"

!sed -i "1s@onnx_config.py']@onnx_config.py', '../../_base_/backends/onnxruntime.py']@" {config_path}

Select Model

This notebook can experiment the following model.

• Mono detection (Camera model)
  • pgd_r101_caffe_fpn_gn_kitti
  • smoke_dla34_dlaneck_gn_kitti
• Voxel detection (Lidar model)
  • pointpillars_hv_fpn_sbn_nus_3d
  • pointpillars_hv_regnet_1_6gf_fpn_hfa_sbn_nus_3d
  • pointpillars_hv_regnet_1_6gf_fpn_hfa_sbn_strong_aug_nus_3d
  • pointpillars_hv_regnet_3_2gf_fpn_hfa_sbn_nus_3d
  • pointpillars_hv_regnet_3_2gf_fpn_hfa_sbn_strong_aug_nus_3d
  • pointpillars_hv_regnet_400mf_fpn_hfa_sbn_nus_3d

Now we are going to experiment pointpillars_hv_regnet_1_6gf_fpn_hfa_sbn_strong_aug_nus_3d.

## Select target model.
MODEL_NAME = MMDet3DModelVariant.pointpillars_hv_regnet_1_6gf_fpn_hfa_sbn_strong_aug_nus_3d

mmdet3d_parameters = get_mmdet3d_parameters(MODEL_NAME)

model_input_size = mmdet3d_parameters.model_input_size
mmdeploy_variant = MMDeployVariant.mmdetection3d
mmdeploy_parameters = get_mmdeploy_parameters(mmdeploy_variant, mmdet3d_parameters)

weights = Path(mmdeploy_parameters.weights_url).name

if not Path(weights).exists():
    download_file(mmdeploy_parameters.weights_url, weights)
    print(f"{weights} is downloaded.")

Export Inferencer

In this notebook, inferencer exported with the model name is used for the following processing.

• Create input data for the inference with inferencer.preprocess
• Run neck with inferencer.model.neck for ChimeraNetwork.BACKBONE
• Run head with inferencer.model.bbox_head for ChimeraNetwork.BACKBONE and ChimeraNetwork.WITH_NECK

init_args = {"model": f"configs/{mmdet3d_parameters.config}", "weights": weights}

if mmdet3d_parameters.task == MMDet3DTask.LidarDetection:
    inferencer = LidarDet3DInferencer(**init_args)
elif mmdet3d_parameters.task == MMDet3DTask.MonoDetection:
    inferencer = MonoDet3DInferencer(**init_args)
else:
    raise ValueError(f"{mmdet3d_parameters.task } is not implemented")

Export ONNX

os.chdir(Path(CUR_DIR) / "mmdeploy")

from mmdeploy.apis import torch2onnx

os.chdir(CUR_DIR)

onnx_file = f"{MODEL_NAME}.onnx"

if mmdet3d_parameters.task == MMDet3DTask.LidarDetection:
    input_data = "../../../common/calibration/face/daniel_maverick.png"  # dummy
elif mmdet3d_parameters.task == MMDet3DTask.MonoDetection:
    kitti_datasample = KittiDataSample()
    input_data = kitti_datasample.infos
else:
    raise ValueError(f"{mmdet3d_parameters.task} is not implemented")


## Convert the model to ONNX
torch2onnx(
    input_data,
    work_dir=".",
    save_file=onnx_file,
    deploy_cfg=mmdeploy_parameters.deploy_config,
    model_cfg=mmdeploy_parameters.config,
    model_checkpoint=weights,
    device="cpu",
)

onnx_model = onnx_check_and_simplify(onnx.load(onnx_file), mmdet3d_parameters.skip_shape_inference)
onnx.save(onnx_model, onnx_file)

print(f"ONNX is exported and simplified as {onnx_file}")

Split ONNX

deploy_config = mmengine.Config.fromfile(
    f"mmdeploy/configs/mmdet3d/{mmdet3d_parameters.deploy_config}"
)

input_edges = deploy_config.onnx_config.input_names
output_edges = deploy_config.onnx_config.output_names

if mmdet3d_parameters.chimera_network is not None:
    target_onnx_file, second_onnx_file = split_onnx(
        onnx_file,
        mmdet3d_parameters,
        input_edges=input_edges,
        output_edges=output_edges,
    )
else:
    split_edges = mmdet3d_parameters.split_edges

    onnx_model = onnx.load(onnx_file)
    feature_extractor_onnx_file = f"{Path(onnx_file).stem}-feature_extractor.onnx"
    target_onnx_file = f"{Path(onnx_file).stem}-predictor.onnx"

    sub_graph = cut_onnx(onnx_model, input_edges, split_edges)
    onnx.save(sub_graph, feature_extractor_onnx_file)
    print(f"Feature extractor onnx saved as {feature_extractor_onnx_file}")

    sub_graph = cut_onnx(onnx_model, split_edges, output_edges)
    sub_graph = onnx_check_and_simplify(sub_graph, mmdet3d_parameters.skip_shape_inference)
    onnx.save(sub_graph, target_onnx_file)
    print(f"Predictor onnx is saved as {target_onnx_file}")

Quantization

We will use sample data under demo/data which comes with mmdetection3d repository.

mh = ModelHelperDetector3D(
    inferencer=inferencer,
    task=mmdet3d_parameters.task,
)

quantized_results = quadric_quantize(
    target_onnx_file,
    num_images=1,
    mh=mh,
    calibration_folder=f"demo/data/{mmdet3d_parameters.dataset}",
    asymmetric_activation=True,
)
print(
    f"quantized onnx: {quantized_results.qmodel_path}, tranges file: {quantized_results.tranges_path}"
)

Compilation

The above quantization is done with the sample data, of which number is only one. We observe that MMDet3DModelVariant.pointpillars_hv_fpn_sbn_nus_3d does not get enough accuracy with it. For users convenience, we prepared a onnx file which was quantized with more calibration data for this model.

prefix_dir = (
    Path("quant_files")
    if MODEL_NAME == MMDet3DModelVariant.pointpillars_hv_fpn_sbn_nus_3d
    else Path(".")
)

quant_onnx_file = str(prefix_dir / Path(quantized_results.qmodel_path).name)
tranges_file = str(prefix_dir / Path(quantized_results.tranges_path).name)

cgc_job = ChimeraJob(
    model_p=quant_onnx_file,
    trange_file=tranges_file,
)
cgc_job.compile(quiet=True)
print(cgc_job)

Demo

3D Detection models have at least 7 elements in the their outputs.

• x: Box center position in the x-axis (m)
• y: Box center position in the y-axis (m)
• z: Box center position in the z-axis (m)
• dx: Box length (m)
• dy: Box width (m)
• dz: Box height (m)
• yaw: Rotation angle around the z-axis (rad)

pointpillars_hv_fpn_sbn_nus_3d has the following additional elements. Therefore, the total number of elements is 9. In this notebook, we don't use them.

• vx: Velocity along the x-axis (m/s)
• vy: Velocity along the y-axis (m/s)

Inference

if mmdet3d_parameters.dataset == "kitti":
    kitti = KittiDataSample()
    sample_info = get_kitti_sample()
    input_path = kitti.img
elif mmdet3d_parameters.dataset == "nuscenes":
    sample_info = get_nuscene_sample()
    input_path = sample_info.lidar_path
else:
    raise ValueError(f"{mmdet3d_parameters.dataset} is not implemented")

engines = {
    InferenceEngine.CHIMERA_ORT_INT8: cgc_job,
    InferenceEngine.CHIMERA_ISS_INT8: cgc_job,
}

outputs_per_inference_results = {}
for inference_engine, engine in engines.items():
    outputs_per_inference_results[inference_engine] = perform_mmdet3d_inference(
        inference_engine, engine, input_path, inferencer, mh, mmdet3d_parameters
    )

Run Statistics

print(cgc_job)
cgc_job.plot_run_statistics()

Display Inference Results

Display 3D Boxes Projected on Image

As noted in mmdetection3d doc page, the prediction score of PGD is not among (0, 1). In case this model is selected, we don't display scores and labels for pgd_r101_caffe_fpn_gn_kitti to avoid confusion.

if MODEL_NAME == MMDet3DModelVariant.pgd_r101_caffe_fpn_gn_kitti:
    show_class = show_score = False
else:
    show_class = show_score = True

pic_len = len(engines) + 1

ax, idx = {}, 1
fig = plt.figure(figsize=(5 * pic_len, 5), tight_layout=True)

ax[idx] = fig.add_subplot(int("1%s%s" % (pic_len, idx)))
ax[idx].imshow(Image.open(sample_info.image_path))
ax[idx].set_title("Original Image")
ax[idx].axis("off")

for inference_engine, results_dict in outputs_per_inference_results.items():
    idx += 1
    ax[idx] = fig.add_subplot(int("1%s%s" % (pic_len, idx)))
    ax[idx] = draw_projected_bboxes(
        results_dict["predictions"][0],
        ax[idx],
        sample_info,
        inferencer.model.cfg["class_names"],
        mmdet3d_parameters.visual_threshold,
        show_class=show_class,
        show_score=show_score,
    )
    ax[idx].set_title(f"{MODEL_NAME}\n{str(inference_engine).upper()}")
    ax[idx].axis("off")

fig.show()

Display 3D Boxes on Bird's Eye View

ax, idx = {}, 1
fig = plt.figure(figsize=(5 * pic_len, 5), tight_layout=True)
extent = get_imshow_extent(mmdet3d_parameters.dataset, sample_info)

ax[idx] = fig.add_subplot(int("1%s%s" % (pic_len, idx)))
bev = get_bev_image(sample_info.points.T, sample_info)
ax[idx].imshow(bev, extent=extent)
ax[idx].set_title(f"Bird's Eye View ({sample_info.cam_type})")
ax[idx].set_xlabel("Horizontal direction (m)")
ax[idx].set_ylabel("Forward direction (m)")

for inference_engine, results_dict in outputs_per_inference_results.items():
    bev_img = draw_bev_image(
        results_dict["predictions"][0],
        sample_info,
        inferencer.model.cfg["class_names"],
        mmdet3d_parameters.visual_threshold,
        show_class=show_class,
        show_score=show_score,
    )
    idx += 1
    ax[idx] = fig.add_subplot(int("1%s%s" % (pic_len, idx)))
    ax[idx].imshow(bev_img, extent=extent)
    ax[idx].set_title(f"{MODEL_NAME}\n{str(inference_engine).upper()}")
    ax[idx].set_xlabel("Horizontal direction (m)")
    ax[idx].set_ylabel("Forward direction (m)")

fig.show()

Citation

@misc{mmdet3d2020,
    title={{MMDetection3D: OpenMMLab} next-generation platform for general {3D} object detection},
    author={MMDetection3D Contributors},
    howpublished = {\url{https://github.com/open-mmlab/mmdetection3d}},
    year={2020}
}