NOTE: The Jupyter Notebook below is included in the Chimera SDK and can be run interactively by running the following CLI command:
$ quadric sdk notebook
From the Jupyter Notebook window in your browser, select the notebook named /quadric/sdk-cli/examples/models/mediapipe/face/MediapipeFace.ipynb.
TFLITE Mediapipe Face Solution
MediaPipe Face is a solution that detects face positions and estimates 468 3D face landmarks for each face in real-time even on mobile devices. It employs machine learning (ML) to infer the 3D facial surface, requiring only a single camera input without the need for a dedicated depth sensor. Utilizing lightweight model architectures together with GPU acceleration throughout the pipeline, the solution delivers real-time performance critical for live experiences.
Face Pipeline
Mediapipe face pipeline consists of two real-time deep neural network models that work together: A detector that operates on the full image and computes face locations and a face landmark model that operates on those locations and predicts the approximate surface via regression. Having the face accurately cropped drastically reduces the need for common data augmentations like affine transformations consisting of rotations, translation and scale changes. Instead it allows the network to dedicate most of its capacity towards coordinate prediction accuracy.
The following figure shows a face solution example. When the face detector model takes an original image in the left as an input, it detects two faces like the middle images.
The face landmark model takes those faces each by each as an input, and infers face landmakrs as shown in the right.
Demo
Now, we are going to use actual Mediapipe models to infer face positions and face landmarks.
- Models
- Face Detection Model
- In this notebook, face_detection_full_range is used.
- Face Landmark Model
- In this notebook, face_landmark is used.
- The landmark network receives as input a cropped video frame without additional depth input. The model outputs the positions of the 3D points, as well as the probability of a face being present.
- Face Detection Model
Convert TFLITE to ONNX
We are going to convert TFLITE files to ONNX files.
The following packages are installed for this conversion.
- tensorflow
- onnxconverter-common
- tensorflow-onnx
!pip3 install -r ../../../requirements.txt -q
[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv[0m[33m
[0m
import os
import sys
from pathlib import Path
FILE_PATH = Path(os.path.abspath(""))
sys.path.append(f"{FILE_PATH.parent}")
from examples.models.zoo.zoo_utils import download_and_extract_zip
github_list = [
(
"https://github.com/microsoft/onnxconverter-common/archive/refs/tags/v%s.zip",
"1.15.0",
),
("https://github.com/onnx/tensorflow-onnx/archive/refs/tags/v%s.zip", "1.16.1"),
]
for url, version in github_list:
url = url % (version)
basename = url.split("/")[-5]
filename = Path(url).name
if not Path(basename).exists():
download_and_extract_zip(url, filename)
Path(f"{basename}-{version}").rename(basename)
Path(filename).unlink(missing_ok=True)
print(f"{basename} is setup.")
onnxconverter-common is setup.
tensorflow-onnx is setup.
The following TFLITE files are converted to ONNX files.
- face_detection_full_range.tflite
- face_landmark.tflite
from pathlib import Path
from mputils.convert_utils import create_onnx_from_tflite
tf_file_list = ["face_landmark.tflite", "face_detection_full_range.tflite"]
for tf_file in tf_file_list:
file = create_onnx_from_tflite(tf_file)
file = Path(file)
new_file = f"{file.stem}_float32{file.suffix}".replace("-convert", "")
os.rename(file, new_file)
print(f"onnx is renamed to {new_file}")
Generate onnx as face_landmark-convert.onnx
onnx is renamed to face_landmark_float32.onnx
Generate onnx as face_detection_full_range-convert.onnx
onnx is renamed to face_detection_full_range_float32.onnx
Compilation of each network
At first, those two models are compiled with ChimeraJob().
from pathlib import Path
import subprocess
from tvm.contrib.epu.chimera_job.quantize import quadric_quantize
import tvm.contrib.epu.chimera_job.core as core
from tvm.contrib.epu.chimera_job.hw_config import (
DEFAULT_8_ARRAY_SIZE,
DEFAULT_16_ARRAY_SIZE,
)
from tvm.contrib.epu.chimera_job.chimera_job import ChimeraJob
from sdk_cli.utils import model_helpers
def get_cgc_job(
onnx_model,
input_size,
calibration_folder="images",
hw_config=DEFAULT_8_ARRAY_SIZE,
):
dataset_input_size = (input_size, input_size) # an (W, H) tuple
dataset_mean = [0, 0, 0]
dataset_std = [1, 1, 1]
# include quadric's cli helpers and instantiate a module to help
mh = model_helpers.ModelHelper(dataset_input_size, dataset_mean, dataset_std)
print("Start quantizing %s" % (onnx_model))
quantize_result = quadric_quantize(
onnx_model, 100, mh, calibration_folder, asymmetric_activation=True
)
print("Define chimerajob with %s" % (quantize_result.qmodel_path))
cgc_job = ChimeraJob(
model_p=quantize_result.qmodel_path,
**hw_config.to_dict(),
trange_file=quantize_result.tranges_path,
)
print("Analyze network")
cgc_job.analyze_network()
print("Start network compilation")
cgc_job.compile(quiet=True)
return cgc_job, quantize_result.qmodel_path, mh
face_detection_full_range
MediaPipe Face Detection Full Range is a face detection solution based on BlazeFace, a relatively lightweight model for detecting single or multiple faces within images from a smartphone camera or webcam. The model is optimized for full-range images, like those taken with a back-facing phone camera images. The model architecture uses a technique similar to a CenterNet convolutional network with a custom encoder.
onnx_file = "face_detection_full_range_float32.onnx"
face_detect_full_cgc, qmodel_path, mh_detect_full = get_cgc_job(
onnx_file,
input_size=192,
calibration_folder="../../../common/calibration/face",
)
print(face_detect_full_cgc)
/tmp/ipykernel_26051/2519288433.py:25: DeprecationWarning: Call to deprecated class ModelHelper. ('ModelHelper' class is being deprecated. Quadric APIs have been updated to use PyTorch datasets and transforms instead.) -- Deprecated since version 24.01.
mh = model_helpers.ModelHelper(dataset_input_size, dataset_mean, dataset_std)
Start quantizing face_detection_full_range_float32.onnx
2026-06-19 03:58 - INFO - epu - quantize - Collecting calibration data
2026-06-19 03:58 - INFO - epu - quantize - Optimized model to opset
2026-06-19 03:58 - INFO - epu - quantize - Saved optimized model to face_detection_full_range_float32_float32_opt.onnx
2026-06-19 03:58 - INFO - epu - quantize - Input shapes: [1, 3, 192, 192]. Input names: inputs0
2026-06-19 03:58 - INFO - epu - quantize - Output shapes: [[1, 2304, 16], [1, 2304, 1]]. Output names: ['outputs0', 'outputs1']
2026-06-19 03:58 - INFO - epu - quantize - applying calibration data to input: inputs0
2026-06-19 03:58 - INFO - epu - quantize - calibration set size: 8
2026-06-19 03:58 - INFO - epu - quantize - Running real quantization on this input: inputs0 with input shape: [1, 3, 192, 192]
2026-06-19 03:58 - DEBUG - epu - quantize - Full exclusion set for quantization: ['Softmax', 'Sigmoid', 'QuadricCustomOp']
2026-06-19 03:58 - DEBUG - epu - quantize - excl_nodes []
2026-06-19 03:58 - INFO - epu - quantize - Quantization started...
WARNING:root:Please use QuantFormat.QDQ for activation type QInt8 and weight type QInt8. Or it will lead to bad performance on x64.
2026-06-19 03:58 - INFO - epu - quantize - Quantization done succesfully!
2026-06-19 03:58 - INFO - epu - quantize - ONNX full precision model size: 1.99 MB
2026-06-19 03:58 - INFO - epu - quantize - ONNX quantized model size: 0.58 MB
2026-06-19 03:58 - INFO - epu - quantize - Saved quantized model to /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx
2026-06-19 03:58 - INFO - epu - quantize - Saved shape inferenced model to /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx
2026-06-19 03:58 - INFO - epu - quantize - Checking for remaining FLOAT/FLOAT16 types.
2026-06-19 03:58 - INFO - epu - quantize - Model still has FLOAT/FLOAT16 types. Creating ranges for floating point tensors using calibration data
2026-06-19 03:58 - INFO - epu - quantize - Saved tensor ranges to /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx.tranges
Define chimerajob with /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx
/tmp/ipykernel_26051/2519288433.py:33: DeprecationWarning: Specifying hardware configuration through individual parameters is deprecated. Please use hw_config parameter instead. Example: hw_cfg = HWConfig(product='QC-U', ocm_size='16MB'); ChimeraJob('model.onnx', hw_config=hw_cfg)
cgc_job = ChimeraJob(
Analyze network
2026-06-19 03:58 - INFO - epu - chimera_job - START==================================onnx_ingest
2026-06-19 03:58 - INFO - epu - chimera_job - Numerical ranges provided
2026-06-19 03:58 - INFO - epu - codegen - START===============================optimize_relay
2026-06-19 03:58 - INFO - epu - codegen - START====================quantize_to_cpu_runnable_fx
2026-06-19 03:58 - INFO - epu - fx -
Source name Op Output 0 Range Output 0 Frac Bits
---------------------------------- -------------------------- --------------------- --------------------
regressor_face_4_DequantizeLinear contrib.epu.dequantize [-55.5684f, 74.9649f] 24
reshaped_regressor_face_4 reshape [-55.5684f, 74.9649f] 24
classifier_face_4_DequantizeLinear contrib.epu.qlinear_conv2d [-10.9f, 2.85989f] 27
reshaped_classifier_face_4 reshape [-10.9f, 2.85989f] 27
Analysis of /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx
Start network compilation
╒═════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════╕
│ Module Name │ face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ONNX File │ /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Product Target │ QC-N │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Number of Cores │ 1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ISS Clock Frequency │ 1.700 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ L2M Size │ 8MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ LRM Size │ 4kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ External Read BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ External Write BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ MACS per PE │ 16 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max L2M │ 0.816MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max LRM │ 1.625kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max Temp Ext Bytes │ 0.000MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Network GMACs │ 0.106 │
╘═════════════════════╧════════════════════════════════════════════════════════════════════════════════════════════════╛
╒════╤════════╤══════════╤══════════════════╤══════════════════════════╤═══════╕
│ │ Type │ Name │ shape │ type │ mse │
╞════╪════════╪══════════╪══════════════════╪══════════════════════════╪═══════╡
│ 0 │ Input │ inputs0 │ [1, 3, 192, 192] │ tensor[FixedPoint32<30>] │ n/a │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 1 │ Output │ outputs0 │ [1, 2304, 16] │ tensor[FixedPoint32<24>] │ n/a │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 2 │ Output │ outputs1 │ [1, 2304, 1] │ tensor[FixedPoint32<27>] │ n/a │
╘════╧════════╧══════════╧══════════════════╧══════════════════════════╧═══════╛
face_landmark
MediaPipe Face Landmark is a solution that estimates face landmarks in real-time even on mobile devices. It employs machine learning (ML) to infer the facial surface, requiring only a single camera input without the need for a dedicated depth sensor. Utilizing lightweight model architectures together with AI acceleration throughout the pipeline, the solution delivers real-time performance critical for live experiences.
onnx_file = "face_landmark_float32.onnx"
face_landmark_cgc, qmodel_path, mh_landmark = get_cgc_job(
onnx_file,
input_size=192,
calibration_folder="../../../common/calibration/face",
hw_config=DEFAULT_16_ARRAY_SIZE,
)
print(face_landmark_cgc)
Start quantizing face_landmark_float32.onnx
2026-06-19 04:02 - INFO - epu - quantize - Collecting calibration data
2026-06-19 04:02 - INFO - epu - quantize - Optimized model to opset
2026-06-19 04:02 - INFO - epu - quantize - Saved optimized model to face_landmark_float32_float32_opt.onnx
2026-06-19 04:02 - INFO - epu - quantize - Input shapes: [1, 3, 192, 192]. Input names: inputs0
2026-06-19 04:02 - INFO - epu - quantize - Output shapes: [[1, 1404, 1, 1], [1, 1, 1, 1]]. Output names: ['outputs0', 'outputs1']
2026-06-19 04:02 - INFO - epu - quantize - applying calibration data to input: inputs0
2026-06-19 04:02 - INFO - epu - quantize - calibration set size: 8
2026-06-19 04:02 - INFO - epu - quantize - Running real quantization on this input: inputs0 with input shape: [1, 3, 192, 192]
2026-06-19 04:02 - INFO - epu - quantize - Quantization started...
WARNING:root:Please use QuantFormat.QDQ for activation type QInt8 and weight type QInt8. Or it will lead to bad performance on x64.
2026-06-19 04:02 - INFO - epu - quantize - Quantization done succesfully!
2026-06-19 04:02 - INFO - epu - quantize - ONNX full precision model size: 2.32 MB
2026-06-19 04:02 - INFO - epu - quantize - ONNX quantized model size: 0.64 MB
2026-06-19 04:02 - INFO - epu - quantize - Saved quantized model to /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx
2026-06-19 04:02 - INFO - epu - quantize - Saved shape inferenced model to /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx
2026-06-19 04:02 - INFO - epu - quantize - Checking for remaining FLOAT/FLOAT16 types.
2026-06-19 04:02 - INFO - epu - quantize - Model still has FLOAT/FLOAT16 types. Creating ranges for floating point tensors using calibration data
2026-06-19 04:02 - INFO - epu - quantize - Saved tensor ranges to /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx.tranges
Define chimerajob with /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx
/tmp/ipykernel_26051/2519288433.py:33: DeprecationWarning: Specifying hardware configuration through individual parameters is deprecated. Please use hw_config parameter instead. Example: hw_cfg = HWConfig(product='QC-U', ocm_size='16MB'); ChimeraJob('model.onnx', hw_config=hw_cfg)
cgc_job = ChimeraJob(
Analyze network
2026-06-19 04:02 - INFO - epu - chimera_job - START==================================onnx_ingest
2026-06-19 04:02 - INFO - epu - chimera_job - Numerical ranges provided
2026-06-19 04:02 - INFO - epu - codegen - START===============================optimize_relay
2026-06-19 04:02 - INFO - epu - codegen - START====================quantize_to_cpu_runnable_fx
2026-06-19 04:02 - INFO - epu - fx -
Source name Op Output 0 Range Output 0 Frac Bits
-------------------------- -------------------------- ---------------------- --------------------
conv2d_1_DequantizeLinear contrib.epu.qlinear_conv2d [-6.0632f, 5.47498f] 28
p_re_lu_1 nn.prelu [-4.54444f, 5.47498f] 28
add_1_DequantizeLinear contrib.epu.dequantize [-8.15922f, 8.9684f] 27
p_re_lu_2 nn.prelu [-1.12671f, 8.9684f] 27
add_2_DequantizeLinear contrib.epu.dequantize [-16.8298f, 9.02322f] 26
p_re_lu_3 nn.prelu [-3.79051f, 9.02322f] 26
add_3_DequantizeLinear contrib.epu.dequantize [-8.20395f, 9.01756f] 27
p_re_lu_4 nn.prelu [-3.93883f, 9.01756f] 27
add_4_DequantizeLinear contrib.epu.dequantize [-10.2372f, 7.9464f] 27
p_re_lu_5 nn.prelu [-1.86943f, 7.9464f] 27
add_5_DequantizeLinear contrib.epu.dequantize [-8.91595f, 10.2004f] 27
p_re_lu_6 nn.prelu [-1.70362f, 10.2004f] 27
add_6_DequantizeLinear contrib.epu.dequantize [-5.343f, 10.3717f] 27
p_re_lu_7 nn.prelu [-2.30221f, 10.3717f] 27
add_7_DequantizeLinear contrib.epu.dequantize [-5.61424f, 10.0432f] 27
p_re_lu_8 nn.prelu [-2.33847f, 10.0432f] 27
add_8_DequantizeLinear contrib.epu.dequantize [-5.90009f, 10.0238f] 27
p_re_lu_9 nn.prelu [-4.17267f, 10.0238f] 27
add_9_DequantizeLinear contrib.epu.dequantize [-3.73502f, 9.90592f] 27
p_re_lu_10 nn.prelu [-1.28391f, 9.90592f] 27
add_10_DequantizeLinear contrib.epu.dequantize [-5.17593f, 9.42135f] 27
p_re_lu_11 nn.prelu [-2.27727f, 9.42135f] 27
add_11_DequantizeLinear contrib.epu.dequantize [-6.74201f, 11.7252f] 27
p_re_lu_12 nn.prelu [-1.0629f, 11.7252f] 27
add_12_DequantizeLinear contrib.epu.dequantize [-3.65829f, 12.5794f] 27
p_re_lu_13 nn.prelu [-2.14738f, 12.5794f] 27
add_13_DequantizeLinear contrib.epu.dequantize [-5.04747f, 12.8016f] 27
p_re_lu_14 nn.prelu [-2.56531f, 12.8016f] 27
add_14_DequantizeLinear contrib.epu.dequantize [-5.4299f, 15.1551f] 26
p_re_lu_15 nn.prelu [-1.73301f, 15.1551f] 26
add_15_DequantizeLinear contrib.epu.dequantize [-3.99081f, 15.5795f] 26
p_re_lu_16 nn.prelu [-3.15455f, 15.5795f] 26
add_16_DequantizeLinear contrib.epu.dequantize [-4.88799f, 16.1809f] 26
p_re_lu_17 nn.prelu [-2.38132f, 16.1809f] 26
add_17_DequantizeLinear contrib.epu.dequantize [-7.80778f, 18.114f] 26
p_re_lu_18 nn.prelu [-2.21749f, 18.114f] 26
conv2d_19_DequantizeLinear contrib.epu.qlinear_conv2d [-2.71881f, 4.54098f] 28
p_re_lu_19 nn.prelu [-1.12056f, 4.54098f] 28
add_18_DequantizeLinear contrib.epu.dequantize [-3.12041f, 8.06848f] 27
p_re_lu_20 nn.prelu [-1.58519f, 8.06848f] 27
outputs0_DequantizeLinear contrib.epu.qlinear_conv2d [-26.8396f, 162.529f] 23
add_23_DequantizeLinear contrib.epu.dequantize [-3.56931f, 15.7202f] 26
p_re_lu_26 nn.prelu [-0.738514f, 15.7202f] 26
conv2d_29_DequantizeLinear contrib.epu.qlinear_conv2d [-2.90085f, 2.65539f] 29
p_re_lu_27 nn.prelu [-0.988537f, 2.65539f] 29
add_24_DequantizeLinear contrib.epu.dequantize [-2.71781f, 4.71884f] 28
p_re_lu_28 nn.prelu [-0.187392f, 4.71884f] 28
outputs1_DequantizeLinear contrib.epu.qlinear_conv2d [-8.68387f, 1.09035f] 27
Analysis of /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx
Start network compilation
╒═════════════════════╤════════════════════════════════════════════════════════════════════════════════════╕
│ Module Name │ face_landmark_opt_asym_int8_q_QC_P_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ ONNX File │ /quadric/sdk-cli/examples/models/mediapipe/face/face_landmark_opt_asym_int8_q.onnx │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Product Target │ QC-P │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Number of Cores │ 1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ ISS Clock Frequency │ 1.700 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ L2M Size │ 8MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ LRM Size │ 4kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ External Read BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ External Write BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ MACS per PE │ 16 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Max L2M │ 0.533MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Max LRM │ 1.652kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Max Temp Ext Bytes │ 0.000MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
│ Network GMACs │ 0.035 │
╘═════════════════════╧════════════════════════════════════════════════════════════════════════════════════╛
╒════╤════════╤══════════╤══════════════════╤══════════════════════════╤═══════╕
│ │ Type │ Name │ shape │ type │ mse │
╞════╪════════╪══════════╪══════════════════╪══════════════════════════╪═══════╡
│ 0 │ Input │ inputs0 │ [1, 3, 192, 192] │ tensor[FixedPoint32<30>] │ n/a │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 1 │ Output │ outputs0 │ [1, 1404, 1, 1] │ tensor[FixedPoint32<23>] │ n/a │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 2 │ Output │ outputs1 │ [1, 1, 1, 1] │ tensor[FixedPoint32<27>] │ n/a │
╘════╧════════╧══════════╧══════════════════╧══════════════════════════╧═══════╛
Inference
Face Detection
By using the face detection model (face_detection_full_range), face positions are detected from input images with the following patterns.
- Int8 Quantized ONNX: face_detect_full_cgc
- ISS: face_detect_full_cgc
import os
import sys
from pathlib import Path
from typing import Tuple
import onnx
import onnxruntime as nxrun
import matplotlib.pyplot as plt
import numpy as np
import torch
import cv2
import glob
from sdk_cli.node_builtins.classical.normalize import normalize
from sdk_cli.node_builtins.classical.object_detector_postprocessing import (
MEDIAPIPE_FULL_RANGE_FACE_DETECTOR_PARAMETERS,
ObjectDetectorParameters,
object_detector_postprocessing,
)
from sdk_cli.node_builtins.classical.resize_pad import resize_pad
from sdk_cli.node_builtins.inputs.image import load_image
FILE_PATH = Path(os.path.abspath(""))
sys.path.append(f"{FILE_PATH.parent}")
from sdk_cli.lib.inference import InferenceEngine
run_folder = r"../../../common/calibration/face/"
NUM_IMAGES = 3
THREADS = min(NUM_IMAGES, 6)
all_images = glob.glob(run_folder + "*.jpg")[:NUM_IMAGES]
face_detector_parameters: ObjectDetectorParameters = MEDIAPIPE_FULL_RANGE_FACE_DETECTOR_PARAMETERS
engines = {
InferenceEngine.CHIMERA_ORT_INT8: face_detect_full_cgc,
InferenceEngine.CHIMERA_ISS_INT8: face_detect_full_cgc,
}
def preprocess_detector(image_path: str, input_image_size: Tuple[int, int]):
"""Preprocess stuff."""
original_image = load_image(image_path)
input_image, scale, pad = resize_pad(original_image, input_image_size)
# input_image is channels first, usually channels last here
input_image = normalize(input_image, np.array([0.0, 0.0, 0.0]), np.array([1.0, 1.0, 1.0]))
# input_image is channels first and has a batch dimension
assert input_image.shape[1] == 3, "Assertion failed"
assert input_image.shape[2] == input_image_size[1], "Assertion failed"
assert input_image.shape[3] == input_image_size[0], "Assertion failed"
return original_image, input_image.astype(np.float32), scale, pad
def postprocess_detector(
original_image: np.ndarray,
out0: np.ndarray,
out1: np.ndarray,
scale: float,
pad: Tuple[int, int],
object_detector_parametrs: ObjectDetectorParameters,
) -> Tuple[np.ndarray, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Postprocess stuff."""
copy_of_original_image = original_image.copy()
img, affine, box, obj_detections = object_detector_postprocessing(
copy_of_original_image, out0, out1, scale, pad, object_detector_parametrs
)
return (
np.moveaxis(copy_of_original_image, 0, -1),
torch.from_numpy(img),
torch.from_numpy(affine),
torch.from_numpy(box),
torch.from_numpy(obj_detections),
)
outputs_per_inference_engine = dict()
for inference_engine in engines.keys():
all_preprocessed_inputs = []
outputs_per_inference_engine[inference_engine] = []
for index, image in enumerate(all_images):
original_image, transformed_image, scale, pad = preprocess_detector(
image, face_detector_parameters.input_image_size
)
if inference_engine == InferenceEngine.ONNXRUNTIME_FP32:
input_name = engines[inference_engine].get_inputs()[0].name
inference_outputs = engines[inference_engine].run(None, {input_name: transformed_image})
outputs = postprocess_detector(
original_image,
*inference_outputs,
scale=scale,
pad=pad,
object_detector_parametrs=face_detector_parameters,
)
outputs_per_inference_engine[inference_engine].append(outputs)
else: # if inference_engine != InferenceEngine.ONNXRUNTIME_FP32:
all_preprocessed_inputs.append((original_image, transformed_image, scale, pad))
if inference_engine != InferenceEngine.ONNXRUNTIME_FP32:
all_model_inputs = []
for preprocessed_input in all_preprocessed_inputs:
all_model_inputs.append({"inputs0": preprocessed_input[1]})
if inference_engine == InferenceEngine.CHIMERA_ORT_INT8:
all_inference_outputs = engines[inference_engine].run_batch_ort_harness(
inputs=all_model_inputs, threads=THREADS
)
elif inference_engine == InferenceEngine.CHIMERA_ISS_INT8:
all_inference_outputs = engines[inference_engine].run_batch_inference_harness(
inputs=all_model_inputs, threads=THREADS
)
else:
raise ValueError(f"{inference_engine} is not implemented.")
assert len(all_inference_outputs) == len(
all_model_inputs
), f"Length mismatch: {len(all_inference_outputs)} vs {len(all_model_inputs)}."
for index, inference_outputs in enumerate(all_inference_outputs):
original_image = all_preprocessed_inputs[index][0]
scale = all_preprocessed_inputs[index][2]
pad = all_preprocessed_inputs[index][3]
outputs = postprocess_detector(
original_image,
inference_outputs["outputs0"],
inference_outputs["outputs1"],
scale=scale,
pad=pad,
object_detector_parametrs=face_detector_parameters,
)
outputs_per_inference_engine[inference_engine].append(outputs)
2026-06-19 04:06 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|████████████████████████████████████████████| 3/3 [00:00<00:00, 17.93it/s]
Processing: 100%|████████████████████████████████| 3/3 [00:14<00:00, 4.91s/it]
from sdk_cli.node_builtins.outputs.bounding_box_visualizer import (
draw_detections,
draw_roi,
)
def draw_detected_faces(
image: torch.Tensor, boxes: torch.Tensor, detections: torch.Tensor
) -> np.ndarray:
"""Draw detected bounding boxes onto original image."""
image_with_drawings = np.array(image.copy())
draw_roi(image=image_with_drawings, regions_of_interest=boxes)
draw_detections(image=image_with_drawings, detections=detections)
return image_with_drawings
all_opts = dict()
for inference_engine in outputs_per_inference_engine.keys():
all_opts[inference_engine] = []
for outputs in zip(
*[
outputs_per_inference_engine[inference_engine]
for inference_engine in outputs_per_inference_engine.keys()
]
):
ax, idx = dict(), 0
fig = plt.figure(figsize=(4 * len(outputs_per_inference_engine.keys()), 4), tight_layout=True)
for output, inference_engine in zip(outputs, outputs_per_inference_engine.keys()):
idx += 1
ax[idx] = fig.add_subplot(int("1%s%s" % (len(outputs_per_inference_engine.keys()), idx)))
all_opts[inference_engine].append(output)
ax[idx].imshow(draw_detected_faces(output[0].astype(np.uint8), output[3], output[4]))
ax[idx].set_title(f"Face Mesh: {str(inference_engine).upper()}")
ax[idx].axis("off")
fig.show()
print(face_detect_full_cgc)
face_detect_full_cgc.plot_run_statistics()
╒═════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════╕
│ Module Name │ face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ONNX File │ /quadric/sdk-cli/examples/models/mediapipe/face/face_detection_full_range_opt_asym_int8_q.onnx │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Product Target │ QC-N │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Number of Cores │ 1 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ISS Clock Frequency │ 1.700 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ L2M Size │ 8MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ LRM Size │ 4kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ External Read BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ External Write BW │ 128GBps │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ MACS per PE │ 16 │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max L2M │ 0.816MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max LRM │ 1.625kB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Max Temp Ext Bytes │ 0.000MB │
├─────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────┤
│ Network GMACs │ 0.106 │
╘═════════════════════╧════════════════════════════════════════════════════════════════════════════════════════════════╛
╒════╤════════╤══════════╤══════════════════╤══════════════════════════╤═══════╕
│ │ Type │ Name │ shape │ type │ mse │
╞════╪════════╪══════════╪══════════════════╪══════════════════════════╪═══════╡
│ 0 │ Input │ inputs0 │ [1, 3, 192, 192] │ tensor[FixedPoint32<30>] │ n/a │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 1 │ Output │ outputs0 │ [1, 2304, 16] │ tensor[FixedPoint32<24>] │ 0.356 │
├────┼────────┼──────────┼──────────────────┼──────────────────────────┼───────┤
│ 2 │ Output │ outputs1 │ [1, 2304, 1] │ tensor[FixedPoint32<27>] │ 0.031 │
╘════╧════════╧══════════╧══════════════════╧══════════════════════════╧═══════╛
Post-ISS Report 1.7 GHz ***
Fully placed-and-routed gate simulation:
╒══════════════════════════════════╤═════════╕
│ Latency (ms) │ 1.74 │
├──────────────────────────────────┼─────────┤
│ FPS │ 574.96 │
├──────────────────────────────────┼─────────┤
│ Average Power @ 3nm SSGNP (mW) │ 238.81 │
├──────────────────────────────────┼─────────┤
│ FPS per Watt @ 3nm SSGNP (FPS/W) │ 2407.60 │
├──────────────────────────────────┼─────────┤
│ Ext Rd Bytes (MB) │ 1.15 │
├──────────────────────────────────┼─────────┤
│ Ext Wr Bytes (MB) │ 0.15 │
├──────────────────────────────────┼─────────┤
│ Avg Ext Rd BW (GBps) │ 0.64 │
├──────────────────────────────────┼─────────┤
│ Avg Ext Wr BW (GBps) │ 0.08 │
├──────────────────────────────────┼─────────┤
│ MAC Utilization │ 3.49% │
╘══════════════════════════════════╧═════════╛
*** Data generated using 7nm SSGNP gatesim and scaled to 3nm
[SDK-CLI] : TotalCycles: 2,956,730
[SDK-CLI] : Executions/second: 574.96
compute : ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 1.699M
data_array : ▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 638.723K
mac : ▇▇▇▇▇▇▇▇▇▇▇▇▇▇ 488.364K
data_external: ▏ 18.922K
data_ocm : ▇▇▇ 106.151K
for more information check run directory: /quadric/sdk-cli/examples/models/mediapipe/face/ccl_build/face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1/run/20260619_040626_131b51
2026-06-19 04:06 - INFO - epu - chimera_job - Combined plots generated and saved to:
/quadric/sdk-cli/examples/models/mediapipe/face/ccl_build/face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1/run/20260619_040626_131b51/data/face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1.combined.png
'/quadric/sdk-cli/examples/models/mediapipe/face/ccl_build/face_detection_full_range_opt_asym_int8_q_QC_N_1d7_8MB_4kB_128GBps_128GBps_16_OFF_x1_x1/run/20260619_040626_131b51/data'
Face Landmark
By using the detected faces above, face landmakrs are detected with face_landmark. Like the face detection, the following patterns are experimented.
- Int8 Quantized ONNX: face_landmark_cgc
- ISS: face_landmark_cgc
Then, by using the drawing function, those landmarks are drawn on the original images.
from typing import Optional
from PIL import Image
from sdk_cli.node_builtins.classical.denormalize_landmarks import (
denormalize_landmarks,
)
from sdk_cli.node_builtins.classical.object_detector_postprocessing import (
MEDIAPIPE_FACE_DETECTOR_BOUNDING_BOX_REGRESSION_PARAMETERS,
ObjectDetectorParameters,
object_detector_postprocessing,
)
face_regressor_parameters: ObjectDetectorParameters = (
MEDIAPIPE_FACE_DETECTOR_BOUNDING_BOX_REGRESSION_PARAMETERS
)
engines = {
InferenceEngine.CHIMERA_ORT_INT8: face_landmark_cgc,
InferenceEngine.CHIMERA_ISS_INT8: face_landmark_cgc,
}
detector_outputs_for_all_inference_engines = []
for index in range(len(all_images)):
detector_outputs_for_all_inference_engines.append(
{inference_engine: all_opts[inference_engine][index] for inference_engine in engines.keys()}
)
landmark_outputs_per_inference_engine = dict()
for inference_engine in engines.keys():
all_preprocessed_inputs = []
landmark_outputs_per_inference_engine[inference_engine] = []
for detector_output in [
detector_output[inference_engine]
for detector_output in detector_outputs_for_all_inference_engines
]:
original_image, detector_outputs, affines, boxes, detections = detector_output
landmark_outputs, flag_outputs = [], []
if inference_engine == InferenceEngine.ONNXRUNTIME_FP32:
for detector_output, affine in zip(detector_outputs.numpy(), affines):
input_name = engines[inference_engine].get_inputs()[0].name
inference_outputs = engines[inference_engine].run(
None, {input_name: np.expand_dims(detector_output, axis=0)}
)
outputs = denormalize_landmarks(
original_image,
inference_outputs[0],
face_regressor_parameters.input_image_size,
row_size=face_regressor_parameters.row_size,
affines=affine,
)
landmark_outputs.append(outputs[0])
flag_outputs.append(inference_outputs[1][0][0][0])
else: # if inference_engine != InferenceEngine.ONNXRUNTIME_FP32:
all_model_inputs = []
for detector_output in detector_outputs.numpy():
all_model_inputs.append({"inputs0": np.expand_dims(detector_output, axis=0)})
if inference_engine == InferenceEngine.CHIMERA_ORT_INT8:
all_inference_outputs = engines[inference_engine].run_batch_ort_harness(
inputs=all_model_inputs, threads=min(len(all_model_inputs), 6)
)
elif inference_engine == InferenceEngine.CHIMERA_ISS_INT8:
all_inference_outputs = engines[inference_engine].run_batch_inference_harness(
inputs=all_model_inputs, threads=min(len(all_model_inputs), 6)
)
else:
raise ValueError(f"{inference_engine} is not implemented.")
for inference_outputs, affine in zip(all_inference_outputs, affines):
outputs = denormalize_landmarks(
original_image,
inference_outputs["outputs0"],
face_regressor_parameters.input_image_size,
row_size=face_regressor_parameters.row_size,
affines=affine,
)
landmark_outputs.append(outputs[0])
flag_outputs.append(inference_outputs["outputs1"][0][0][0])
landmark_outputs_per_inference_engine[inference_engine].append(
(
original_image,
np.stack(landmark_outputs),
np.stack(flag_outputs),
boxes,
detections,
)
)
2026-06-19 04:06 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|████████████████████████████████████████████| 2/2 [00:00<00:00, 27.61it/s]
2026-06-19 04:06 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|████████████████████████████████████████████| 2/2 [00:00<00:00, 22.20it/s]
2026-06-19 04:06 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|████████████████████████████████████████████| 3/3 [00:00<00:00, 19.39it/s]
Processing: 100%|████████████████████████████████| 2/2 [00:03<00:00, 1.73s/it]
Processing: 100%|████████████████████████████████| 2/2 [00:04<00:00, 2.10s/it]
Processing: 100%|████████████████████████████████| 3/3 [00:04<00:00, 1.43s/it]
from typing import List, Optional
from sdk_cli.node_builtins.outputs.bounding_box_visualizer import (
draw_detections,
draw_landmarks,
draw_roi,
)
def draw_detected_face_landmarks(
original_image: torch.Tensor,
all_landmarks_per_image: torch.Tensor,
all_flags_per_image: torch.Tensor,
connections: List[Tuple[int, int]],
draw_threshold: float,
boxes: Optional[torch.Tensor] = None,
detections: Optional[torch.Tensor] = None,
landmark_cut: Optional[int] = None,
) -> np.ndarray:
"""Draw for fun."""
image_with_drawings = np.array(original_image.copy())
if len(image_with_drawings.shape) == 4:
image_with_drawings = image_with_drawings[0]
if image_with_drawings.shape[0] == 3:
image_with_drawings = np.moveaxis(image_with_drawings, 0, -1)
image_with_drawings = np.ascontiguousarray(image_with_drawings, dtype=np.uint8)
for all_landmarks_per_detection, all_flags_per_detection in zip(
all_landmarks_per_image, all_flags_per_image
):
for landmark, flag in zip(
np.expand_dims(all_landmarks_per_detection, 0), all_flags_per_detection
):
if flag > draw_threshold:
landmark_to_draw = (
all_landmarks_per_detection[:landmark_cut, :]
if landmark_cut
else all_landmarks_per_detection
)
draw_landmarks(
image=image_with_drawings,
points=landmark_to_draw[:, :2],
connections=connections,
)
if boxes is not None:
draw_roi(image=image_with_drawings, regions_of_interest=boxes)
if detections is not None:
draw_detections(image=image_with_drawings, detections=detections)
return image_with_drawings
for landmark_outputs in zip(
*[
landmark_outputs_per_inference_engine[inference_engine]
for inference_engine in landmark_outputs_per_inference_engine.keys()
]
):
ax, idx = dict(), 0
fig = plt.figure(
figsize=(4 * len(landmark_outputs_per_inference_engine.keys()), 4),
tight_layout=True,
)
for output, inference_engine in zip(
landmark_outputs, landmark_outputs_per_inference_engine.keys()
):
idx += 1
ax[idx] = fig.add_subplot(
int("1%s%s" % (len(landmark_outputs_per_inference_engine.keys()), idx))
)
ax[idx].imshow(
draw_detected_face_landmarks(
output[0].astype(np.uint8),
output[1],
output[2],
face_regressor_parameters.connections,
face_regressor_parameters.draw_threshold,
boxes=output[3],
detections=output[4],
)
)
ax[idx].set_title(f"Face Mesh: {str(inference_engine).upper()}")
ax[idx].axis("off")
fig.show()
(Option) Standalone Face Landmark
This section shows the capability of the standalone face landmark model by feeding detected face images directly. To run command cells, please set True to run_option.
run_option = False
if run_option:
from sdk_cli.node_builtins.classical.resize.resize import one_step_resize
run_folder = r"../images/face/"
all_images = [
Path(run_folder) / "detected_face0.jpg",
Path(run_folder) / "detected_face1.jpg",
Path(run_folder) / "detected_face2.jpg",
Path(run_folder) / "detected_face4.jpg",
]
THREADS = min(len(all_images), 6)
def preprocess_landmark(image_path: str, input_image_size: Tuple[int, int]):
"""Preprocess some stuff."""
original_image = load_image(image_path)
transformed_image = one_step_resize(
np.moveaxis(original_image, 0, -1), (192, 192, 3)
).astype(np.float32)
transformed_image = normalize(
np.moveaxis(transformed_image, -1, 0),
np.array([0.0, 0.0, 0.0]),
np.array([1.0, 1.0, 1.0]),
)
return original_image, transformed_image
face_regressor_parameters: ObjectDetectorParameters = (
MEDIAPIPE_FACE_DETECTOR_BOUNDING_BOX_REGRESSION_PARAMETERS
)
engines = {
InferenceEngine.CHIMERA_ORT_INT8: face_landmark_cgc,
InferenceEngine.CHIMERA_ISS_INT8: face_landmark_cgc,
}
landmark_outputs_per_inference_engine = dict()
for inference_engine in engines.keys():
all_original_images, all_model_inputs = [], []
landmark_outputs_per_inference_engine[inference_engine] = []
for index, image in enumerate(all_images):
original_image, transformed_image = preprocess_landmark(
image, face_detector_parameters.input_image_size
)
if inference_engine == InferenceEngine.ONNXRUNTIME_FP32:
input_name = engines[inference_engine].get_inputs()[0].name
inference_outputs = engines[inference_engine].run(
None, {input_name: transformed_image}
)
outputs = denormalize_landmarks(
original_image,
inference_outputs[0],
face_regressor_parameters.input_image_size,
row_size=face_regressor_parameters.row_size,
)
landmark_outputs_per_inference_engine[inference_engine].append(
(original_image, outputs, inference_outputs[1][0][0], None, None)
)
else: # if inference_engine != InferenceEngine.ONNXRUNTIME_FP32:
all_model_inputs.append({"inputs0": transformed_image})
all_original_images.append(original_image)
if inference_engine != InferenceEngine.ONNXRUNTIME_FP32:
if inference_engine == InferenceEngine.CHIMERA_ORT_INT8:
all_inference_outputs = engines[inference_engine].run_batch_ort_harness(
inputs=all_model_inputs, threads=THREADS
)
elif inference_engine == InferenceEngine.CHIMERA_ISS_INT8:
all_inference_outputs = engines[inference_engine].run_batch_inference_harness(
inputs=all_model_inputs, threads=THREADS
)
else:
raise ValueError(f"{inference_engine} is not implemented.")
for original_image, inference_outputs in zip(
all_original_images, all_inference_outputs
):
outputs = denormalize_landmarks(
original_image,
inference_outputs["outputs0"],
face_regressor_parameters.input_image_size,
row_size=face_regressor_parameters.row_size,
)
landmark_outputs_per_inference_engine[inference_engine].append(
(
original_image,
outputs,
inference_outputs["outputs1"][0][0],
None,
None,
)
)
print(face_landmark_cgc)
face_landmark_cgc.plot_run_statistics()
if run_option:
for landmark_outputs in zip(
*[
landmark_outputs_per_inference_engine[inference_engine]
for inference_engine in landmark_outputs_per_inference_engine.keys()
]
):
ax, idx = dict(), 0
fig = plt.figure(
figsize=(4 * len(landmark_outputs_per_inference_engine.keys()), 4),
tight_layout=True,
)
for output, inference_engine in zip(
landmark_outputs, landmark_outputs_per_inference_engine.keys()
):
idx += 1
ax[idx] = fig.add_subplot(
int("1%s%s" % (len(landmark_outputs_per_inference_engine.keys()), idx))
)
ax[idx].imshow(
draw_detected_face_landmarks(
output[0].astype(np.uint8),
output[1],
output[2],
face_regressor_parameters.connections,
face_regressor_parameters.draw_threshold,
boxes=output[3],
detections=output[4],
)
)
ax[idx].set_title(f"Face Mesh: {str(inference_engine).upper()}")
ax[idx].axis("off")
fig.show()
