NOTE: The Jupyter Notebook below is included in the Chimera SDK and can be run interactively by running the following CLI command:
$ quadric sdk notebook
From the Jupyter Notebook window in your browser, select the notebook named /quadric/sdk-cli/examples/models/llama/llama.ipynb.
Llama2 Model Quantization and Evaluationion technique
Llama 2 is a collection of pretrained and fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters. The fine-tuned LLMs, called Llama 2-Chat, are optimized for dialogue use cases. The models outperformed open-source chat models on most benchmarks when it was released, and based on the human evaluations for helpfulness and safety, may be a suitable substitute for closed-source models. For further details, please refer the paper.
- Hugo Touvron, etal. Llama 2: Open Foundation and Fine-Tuned Chat Models
This notebook demonstrates the process of quantizing and evaluating the Llama2 large language model (LLM). It covers:
- Loading pretrained model weights
- Generating ONNX graphs (floating point and quantized)
- Applying custom operators
- Compiling C++ code
- Running inference using ISS (Instruction Set Simulator) and ORT (ONNX Runtime)
- Evaluating model performance:
- Top-1 and Top-5 accuracy
- Perplexity scores
- Output correlation plots
- (Optional) Smooth Quantization technique
import gc
import sys
sys.path.append(".")
from pathlib import Path
from urllib.parse import urlparse, unquote
Set the model size that needs to be tested.
model_size = "15M" # Currently supports 15M only.
Loading predefined model parameters.
from llama_utils import supported_models_dict
from examples.models.zoo.zoo_utils import download_file
model_params = supported_models_dict[model_size]
for k, v in model_params.items():
print(f"{k:>15} : {v}")
checkpoint_url : https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt
sequence_len : 256
dim : 288
vocab_size : 32000
n_heads : 6
n_layers : 6
Download pretrained weights. (if not already downloaded...)
weights_dir = Path.cwd() / "model" / "checkpoints"
weights_dir.mkdir(exist_ok=True)
checkpoint_f_name = unquote(urlparse(model_params["checkpoint_url"]).path).split("/")[-1]
checkpoint_f_path = weights_dir / checkpoint_f_name
if not checkpoint_f_path.is_file():
print("Downloading pretrained weights...")
download_file(model_params["checkpoint_url"], checkpoint_f_path)
print(f"Download completed! (saved to {checkpoint_f_path})")
else:
print(f"{checkpoint_f_path} already exists, skipping download...")
assert checkpoint_f_path.is_file(), f"Failed to load the weights from {weights_dir}"
Downloading pretrained weights...
Download completed! (saved to /quadric/sdk-cli/examples/models/llama/model/checkpoints/stories15M.pt)
Generating ONNX graph of floating point llama model
from model.generate_fp_model import generate_llama_fp_onnx
temp_onnx_dir = Path.cwd() / "llama"
temp_onnx_dir.mkdir(exist_ok=True)
fp_onnx_path = temp_onnx_dir / "llama2_fp32.onnx"
generate_llama_fp_onnx(
str(checkpoint_f_path),
str(fp_onnx_path),
model_params["sequence_len"],
model_params["dim"],
model_params["vocab_size"],
model_params["n_layers"],
model_params["n_heads"],
)
==========================================================================================
Layer (type:depth-idx) Output Shape Param #
==========================================================================================
Transformer [1, 1, 32000] --
├─Embedding: 1-1 [1, 256, 288] 9,216,000
├─Dropout: 1-2 [1, 256, 288] --
├─ModuleList: 1-3 -- --
│ └─TransformerBlock: 2-1 [1, 256, 288] --
│ │ └─RMSNorm: 3-1 [1, 256, 288] 288
│ │ └─Attention: 3-2 -- 331,776
│ │ └─RMSNorm: 3-3 [1, 256, 288] 288
│ │ └─FeedForward: 3-4 -- 663,552
│ └─TransformerBlock: 2-2 [1, 256, 288] --
│ │ └─RMSNorm: 3-5 [1, 256, 288] 288
│ │ └─Attention: 3-6 -- 331,776
│ │ └─RMSNorm: 3-7 [1, 256, 288] 288
│ │ └─FeedForward: 3-8 -- 663,552
│ └─TransformerBlock: 2-3 [1, 256, 288] --
│ │ └─RMSNorm: 3-9 [1, 256, 288] 288
│ │ └─Attention: 3-10 -- 331,776
│ │ └─RMSNorm: 3-11 [1, 256, 288] 288
│ │ └─FeedForward: 3-12 -- 663,552
│ └─TransformerBlock: 2-4 [1, 256, 288] --
│ │ └─RMSNorm: 3-13 [1, 256, 288] 288
│ │ └─Attention: 3-14 -- 331,776
│ │ └─RMSNorm: 3-15 [1, 256, 288] 288
│ │ └─FeedForward: 3-16 -- 663,552
│ └─TransformerBlock: 2-5 [1, 256, 288] --
│ │ └─RMSNorm: 3-17 [1, 256, 288] 288
│ │ └─Attention: 3-18 -- 331,776
│ │ └─RMSNorm: 3-19 [1, 256, 288] 288
│ │ └─FeedForward: 3-20 -- 663,552
│ └─TransformerBlock: 2-6 [1, 256, 288] --
│ │ └─RMSNorm: 3-21 [1, 256, 288] 288
│ │ └─Attention: 3-22 -- 331,776
│ │ └─RMSNorm: 3-23 [1, 256, 288] 288
│ │ └─FeedForward: 3-24 -- 663,552
├─RMSNorm: 1-4 [1, 256, 288] 288
├─Linear: 1-5 [1, 1, 32000] 9,216,000
==========================================================================================
Total params: 24,407,712
Trainable params: 24,407,712
Non-trainable params: 0
Total mult-adds (M): 24.41
==========================================================================================
Input size (MB): 0.00
Forward/backward pass size (MB): 45.08
Params size (MB): 97.63
Estimated Total Size (MB): 142.72
==========================================================================================
/quadric/sdk-cli/examples/models/llama/model/model.py:63: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
assert freqs_cis.shape == (
Saved model to /quadric/sdk-cli/examples/models/llama/llama/llama2_fp32.onnx
Download datasets and Generate quantized ONNX graph
from llama_quantizer import quantize_llama_onnx
quant_onnx_path = temp_onnx_dir / "llama2_optimized_sym_int8_q.onnx"
dataset_path = Path.cwd() / "data"
quantize_llama_onnx(model_params, fp_onnx_path, quant_onnx_path, dataset_path, temp_onnx_dir)
gc.collect()
Compressed dataset file not found at the given path... (/quadric/sdk-cli/examples/models/llama/data/TinyStories_all_data.tar.gz)
Downloading dataset...
Unpacking /quadric/sdk-cli/examples/models/llama/data/TinyStories_all_data.tar.gz...
Using static quantization schema (dataset: json, method: CalibrationMethod.MinMax)
Creating calibrator: CalibrationMethod.MinMax(CalibrationConfig(dataset_name='json', dataset_config_name='default', dataset_split='train', dataset_num_samples=5000, method=<CalibrationMethod.MinMax: 0>, num_bins=None, num_quantized_bins=None, percentile=None, moving_average=False, averaging_constant=0.01))
Collecting tensors statistics...
Computing calibration ranges
Creating static quantizer: QOperator (mode: QLinearOps, schema: s8/s8, channel-wise: False)
Quantizing model...
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_1_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Unsqueeze_18_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_9_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_8_output_0', type is tensor_type {
elem_type: 7
shape {
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_1_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 3
}
}
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_3_output_0', type is tensor_type {
elem_type: 7
shape {
dim {
dim_value: 1
}
}
}
.
Saving quantized model at: /quadric/sdk-cli/examples/models/llama/llama (external data format: False)
Configuration saved in /quadric/sdk-cli/examples/models/llama/llama/ort_config.json
Quantized graph with fixed input shape is saved at /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
Cleaning up temp files...
Done.
4233
Generate Custom-ops replaced ONNX graph
from custom_op_replacer import prepare_custom_ops_graph
custom_op_onnx_path = temp_onnx_dir / "llama2_custom_op_int8_q.onnx"
prepare_custom_ops_graph(model_params, str(quant_onnx_path), str(custom_op_onnx_path))
gc.collect()
Found already existing dataset...
7978
Generate C++ code and compile.
from tvm.contrib.epu.chimera_job.chimera_job import ChimeraJob
cgc_job = ChimeraJob(
model_p=str(custom_op_onnx_path),
onnx_ort_override_p=str(quant_onnx_path),
quiet_iss=False,
)
cgc_job.compile(quiet=True)
2026-09-22 02:50 - INFO - epu - chimera_job - Overriding onnx model ort execution with the onnx model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
Define the number of parallel threads you can run and the total number of runs
num_threads = 6
total_runs = 24
Loading input data from TinyStories dataset
import numpy as np
from llama_utils import load_and_tokenize_validation_data
dataset_path = Path.cwd() / "data"
val_data_array = load_and_tokenize_validation_data(model_params, str(dataset_path), total_runs)
inp_list = [] # must be in int32 type
target_idx_list = []
for inp_toks, target_tok in val_data_array:
inp_list.append({"input": inp_toks.astype(np.int32)})
target_idx_list.append(target_tok)
Downloading validation data...
Download completed! (saved to /quadric/sdk-cli/examples/models/llama/data/TinyStories-valid.txt)
Run multiple instances of ISS and ORT in parallel threads
results_iss_list = cgc_job.run_batch_inference_harness(
inputs=inp_list,
threads=num_threads,
)
results_ort_list = cgc_job.run_batch_ort_harness(
inputs=inp_list,
threads=num_threads,
)
Processing: 100%|██████████████████████████████| 24/24 [03:38<00:00, 9.09s/it]
2026-09-22 02:59 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|██████████████████████████████████████████| 24/24 [00:07<00:00, 3.14it/s]
Calculate Top-1 and Top-5 accuracies
(Compared to ORT output of quantized ONNX)
top1_count = 0
top5_count = 0
for iss, ort in zip(results_iss_list, results_ort_list):
ort_arg_1 = np.argmax(ort["output"].flatten())
iss_arg_1 = np.argmax(iss["output"].flatten())
iss_arg_5 = np.argsort(iss["output"].flatten())[::-1][:5]
if ort_arg_1 == iss_arg_1:
top1_count = top1_count + 1
if ort_arg_1 in iss_arg_5:
top5_count = top5_count + 1
top1_acc = 100 * top1_count / len(results_iss_list)
top5_acc = 100 * top5_count / len(results_iss_list)
print("Accuracy of ISS run. (vs quantized ONNX run)")
print(f"Top-1 Accuracy : {top1_acc:>7.3f}% ({top1_count}/{len(results_iss_list)})")
print(f"Top-5 Accuracy : {top5_acc:>7.3f}% ({top5_count}/{len(results_iss_list)})")
Accuracy of ISS run. (vs quantized ONNX run)
Top-1 Accuracy : 83.333% (20/24)
Top-5 Accuracy : 95.833% (23/24)
Perplexity scores.
import scipy
ort_loss = 0.0
iss_loss = 0.0
for target, iss, ort in zip(target_idx_list, results_iss_list, results_ort_list):
sm_ort = scipy.special.softmax(ort["output"].flatten(), axis=-1)
sm_iss = scipy.special.softmax(iss["output"].flatten(), axis=-1)
ort_loss = ort_loss + (-1 * np.log(sm_ort[target]))
iss_loss = iss_loss + (-1 * np.log(sm_iss[target]))
ort_loss = ort_loss / total_runs
iss_loss = iss_loss / total_runs
ppl_ort = np.exp(ort_loss)
ppl_iss = np.exp(iss_loss)
print("Perplexity Scores. (Lower is better)")
print("Perplexity of ORT run :", ppl_ort)
print("Perplexity of ISS run :", ppl_iss)
Perplexity Scores. (Lower is better)
Perplexity of ORT run : 3.6292876342732123
Perplexity of ISS run : 4.635994369006926
Plot the correlation for each run
Set plot_correlations = True if required...
plot_correlations = False
if plot_correlations:
import matplotlib.pyplot as plt
import math
plt_cols = min(5, total_runs)
plt_rows = math.ceil(total_runs / plt_cols)
fig, ax = plt.subplots(plt_rows, plt_cols, figsize=(20, 3 * plt_rows))
ax = ax.reshape((plt_rows, plt_cols))
i = 0
while i < plt_cols * plt_rows:
plt_c = i % plt_cols
plt_r = i // plt_cols
if i < len(results_iss_list):
ax[plt_r, plt_c].plot(
results_ort_list[i]["output"].flatten(),
results_iss_list[i]["output"].flatten(),
".",
)
ax[plt_r, plt_c].get_xaxis().set_visible(False)
ax[plt_r, plt_c].get_yaxis().set_visible(False)
else:
ax[plt_r, plt_c].set_axis_off()
i = i + 1
plt.show()
Save ISS and ORT output tensors
Set dump_data = True if required...
dump_data = False
if dump_data:
dump_data_dir = Path.cwd() / "data_dumps"
dump_data_dir.mkdir(exist_ok=True)
iss = np.stack([results_iss_list[i]["output"].flatten() for i in range(len(results_ort_list))])
ort = np.stack([results_ort_list[i]["output"].flatten() for i in range(len(results_iss_list))])
np.save(f"{dump_data_dir}/iss_out_batch.npy", iss)
np.save(f"{dump_data_dir}/ort_out_batch.npy", ort)
Text Generation Example
We demonstrate the quantized model running autoregressively and producing stories on ORT and ISS.
Load Vocabulary
from llama_run import load_vocab
import numpy as np
vocab_size = 32000
filename = "model/tokenizer.bin"
vocab = load_vocab(vocab_size, filename)
idx = np.array([[1]]) # Starting token
Quantized Model on ORT
from llama_run import generate_tokens_onnx
import onnxruntime
sess_opt = onnxruntime.SessionOptions()
sess_opt.intra_op_num_threads = 1
session = onnxruntime.InferenceSession(quant_onnx_path, sess_opt)
generated_sequence = generate_tokens_onnx(
session,
idx,
max_new_tokens=256,
vocab=vocab,
do_padding=True,
temperature=0.0,
padding_token=1,
max_seq_len=256,
)
<s> Once upon a time, there was a little girl named Jane was in the forest. She was a very special, but she was a bit scared of the dark and the night when the sun went on the sticks and a night was in the dark. She was scared, but she was also very scared. She was a brave and strong and she could see a light at night.
"Are you a ghost?" Jane, who was her best friend, who had just been looking for her. She had a bright, shiny stone and a big smile on her face.
"Don't be scared, I'm not so scary," she said. "I'm just danc that the sun is pointing at the moon and the stars. It's so beautiful!"
The two of them started to dance and sing and danced around the room. Jane was so happy and she clapped her hands and shouted, "I love you, Jane!"
"I love it, Jane, I'm so glad you're here," she said. "You're the best friend ever, I'm so happy to be here."
Jane and Tom hugged her and said goodbye.
Tokens per second: 4.698140188267626
Quantized Model on ISS
from llama_run import generate_tokens_iss
generated_sequence = generate_tokens_iss(
cgc_job,
idx,
max_new_tokens=10,
vocab=vocab,
do_padding=True,
temperature=0.0,
padding_token=1,
max_seq_len=256,
)
<s>
2026-09-22 03:00 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680dc7a60>
2026-09-22 03:01 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d96650>
2026-09-22 03:01 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:01 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:01 - INFO - epu - iss_testing - Done 0:00:00.185185
Once
2026-09-22 03:01 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d56c38ecb0>
2026-09-22 03:01 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d97520>
2026-09-22 03:01 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:01 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:01 - INFO - epu - iss_testing - Done 0:00:00.359710
upon
2026-09-22 03:01 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d587513fa0>
2026-09-22 03:02 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d56c38ecb0>
2026-09-22 03:02 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:02 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:02 - INFO - epu - iss_testing - Done 0:00:00.201508
a
2026-09-22 03:02 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d587513fa0>
2026-09-22 03:02 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d96650>
2026-09-22 03:02 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:02 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:02 - INFO - epu - iss_testing - Done 0:00:00.212088
time
2026-09-22 03:02 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d587513fa0>
2026-09-22 03:03 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d97520>
2026-09-22 03:03 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:03 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:03 - INFO - epu - iss_testing - Done 0:00:00.169018
,
2026-09-22 03:03 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d56c38ecb0>
2026-09-22 03:03 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d5864e8280>
2026-09-22 03:03 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:03 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:03 - INFO - epu - iss_testing - Done 0:00:00.194413
there
2026-09-22 03:03 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d97520>
2026-09-22 03:04 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d96650>
2026-09-22 03:04 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:04 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:04 - INFO - epu - iss_testing - Done 0:00:00.182049
was
2026-09-22 03:04 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d96650>
2026-09-22 03:04 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d5864e89a0>
2026-09-22 03:04 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:04 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:04 - INFO - epu - iss_testing - Done 0:00:00.163533
a
2026-09-22 03:04 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680dc7a90>
2026-09-22 03:05 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d5864e8460>
2026-09-22 03:05 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:05 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:05 - INFO - epu - iss_testing - Done 0:00:00.159707
little
2026-09-22 03:05 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d96650>
2026-09-22 03:05 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x76d680d97520>
2026-09-22 03:05 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-09-22 03:05 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-09-22 03:05 - INFO - epu - iss_testing - Done 0:00:00.199800
girl
Full generated sequence: <s> Once upon a time, there was a little girl
(Option) Quantizing Llama-15M with Smooth Quantization
Smooth quantization is an accurate and efficient post-training quantization (PTQ) solution for LLMs. Normally, activations in ML models are much harder to quantize than weights due to the presence of outliers. However, different tokens exhibit similar variations across their channels. Based on this observation, SmoothQuant offline migrates the quantization difficulty from activations to weight.
This notebook is for quantizing Llama-15M model using the smooth quantization method.
Further details about smooth quantization can be found in the paper.
- Guangxuan Xiao, Ji Lin, Mickael Seznec, Hao Wu, Julien Demouth, Song Han. SmoothQuant: Accurate and Efficient Post-Training Quantization for Large Language Models
How to Run the Option
In order to run the following, please do the following
- Set
Trueto the followingRUN_OPTION.
RUN_OPTION = False
Define ModelHelpers
import numpy as np
from sdk_cli.utils import model_helpers
from model.tokenizer import Tokenizer
if RUN_OPTION:
tokenizer = Tokenizer()
def tokenize_val(text):
text = text.strip() # get rid of leading/trailing whitespace
tokens = tokenizer.encode(text, bos=True, eos=False) # encode the text, use BOS
np_tokens = np.array(tokens)
return np_tokens.flatten()
mh_val = model_helpers.TextModelHelper(tokenize_val, model_params["sequence_len"])
Download datasets and Generate smooth quantized ONNX graph
from llama_smooth_quantization import llama_smooth_quantization
if RUN_OPTION:
model_size = "15M"
model_params = supported_models_dict[model_size]
fp_onnx_path = temp_onnx_dir / "llama2_fp32.onnx"
smooth_quant_alpha = 0.5
quant_onnx_path = temp_onnx_dir / f"llama2_opt_smooth_quant_{smooth_quant_alpha}.onnx"
dataset_path = Path.cwd() / "data_smq"
temp_onnx_dir = Path.cwd() / "llama"
llama_smooth_quantization(
model_params,
fp_onnx_path,
quant_onnx_path,
smooth_quant_alpha,
dataset_path,
temp_onnx_dir,
)
Run multiple ORT instances in parallel threads
This will run the inputs through both quantized and float32 ONNX graphs...
import multiprocess
import onnxruntime
from tqdm import tqdm
def run_ort_inference(zipped_params):
(fp_onnx_path, q_onnx_path, inp) = zipped_params
def single_ort_inference(onnx_path, inp):
sess_model = onnxruntime.InferenceSession(onnx_path)
onnx_out = sess_model.run([], {"input": inp})[0].flatten()
return onnx_out
fp_out = single_ort_inference(fp_onnx_path, inp)
q_out = single_ort_inference(q_onnx_path, inp)
return (fp_out, q_out)
if RUN_OPTION:
total_runs = 1000
num_threads = 3
val_story_list = mh_val.get_data_list("data_smq/TinyStories-valid.txt")
val_data = mh_val.get_batch_data(val_story_list)
val_d_list = val_data[:total_runs]
total_runs = len(val_d_list)
results_fp_list = []
results_q_list = []
with multiprocess.Pool(num_threads) as p:
zipped_inp = zip([fp_onnx_path] * total_runs, [quant_onnx_path] * total_runs, val_d_list)
results = list(tqdm(p.imap(run_ort_inference, zipped_inp), total=total_runs))
for fp_out, q_out in results:
results_fp_list.append(fp_out)
results_q_list.append(q_out)
Calculate Top-1 and Top-5 accuracies of quantized ONNX graph
(Compared to output of float32 ONNX graph)
if RUN_OPTION:
top1_count = 0
top5_count = 0
for q_out, fp_out in zip(results_q_list, results_fp_list):
fp_arg_1 = np.argmax(fp_out.flatten())
q_arg_1 = np.argmax(q_out.flatten())
q_arg_5 = np.argsort(q_out.flatten())[::-1][:5]
if fp_arg_1 == q_arg_1:
top1_count = top1_count + 1
if fp_arg_1 in q_arg_5:
top5_count = top5_count + 1
top1_acc = 100 * top1_count / len(results_fp_list)
top5_acc = 100 * top5_count / len(results_fp_list)
print(f"Accuracy of Quantized run. (vs FP ONNX run) --- SmoothQuantAlpha: {smooth_quant_alpha}")
print(f"Top-1 Accuracy : {top1_acc:>7.3f}% ({top1_count}/{len(results_fp_list)})")
print(f"Top-5 Accuracy : {top5_acc:>7.3f}% ({top5_count}/{len(results_fp_list)})")
print()
