Model Demo: Llama-2 15M (Baby Llama-2)

NOTE: The Jupyter Notebook below is included in the Chimera SDK and can be run interactively by running the following CLI command:

$ quadric sdk notebook

From the Jupyter Notebook window in your browser, select the notebook named /quadric/sdk-cli/examples/models/llama/llama.ipynb.

Llama2 Model Quantization and Evaluationion technique

Llama 2 is a collection of pretrained and fine-tuned large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters. The fine-tuned LLMs, called Llama 2-Chat, are optimized for dialogue use cases. The models outperformed open-source chat models on most benchmarks when it was released, and based on the human evaluations for helpfulness and safety, may be a suitable substitute for closed-source models. For further details, please refer the paper.

• Hugo Touvron, etal. Llama 2: Open Foundation and Fine-Tuned Chat Models

This notebook demonstrates the process of quantizing and evaluating the Llama2 large language model (LLM). It covers:

1. Loading pretrained model weights
2. Generating ONNX graphs (floating point and quantized)
3. Applying custom operators
4. Compiling C++ code
5. Running inference using ISS (Instruction Set Simulator) and ORT (ONNX Runtime)
6. Evaluating model performance:
   • Top-1 and Top-5 accuracy
   • Perplexity scores
   • Output correlation plots
7. (Optional) Smooth Quantization technique

import gc
import sys

sys.path.append(".")

from pathlib import Path
from urllib.parse import urlparse, unquote

Set the model size that needs to be tested.

model_size = "15M"  # Currently supports 15M only.

Loading predefined model parameters.

from llama_utils import supported_models_dict
from examples.models.zoo.zoo_utils import download_file

model_params = supported_models_dict[model_size]

for k, v in model_params.items():
    print(f"{k:>15} : {v}")

 checkpoint_url : https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt
   sequence_len : 256
            dim : 288
     vocab_size : 32000
        n_heads : 6
       n_layers : 6

Download pretrained weights. (if not already downloaded...)

weights_dir = Path.cwd() / "model" / "checkpoints"
weights_dir.mkdir(exist_ok=True)

checkpoint_f_name = unquote(urlparse(model_params["checkpoint_url"]).path).split("/")[-1]
checkpoint_f_path = weights_dir / checkpoint_f_name

if not checkpoint_f_path.is_file():
    print("Downloading pretrained weights...")
    download_file(model_params["checkpoint_url"], checkpoint_f_path)
    print(f"Download completed! (saved to {checkpoint_f_path})")
else:
    print(f"{checkpoint_f_path} already exists, skipping download...")

assert checkpoint_f_path.is_file(), f"Failed to load the weights from {weights_dir}"

Downloading pretrained weights...
Download completed! (saved to /quadric/sdk-cli/examples/models/llama/model/checkpoints/stories15M.pt)

Generating ONNX graph of floating point llama model

from model.generate_fp_model import generate_llama_fp_onnx

temp_onnx_dir = Path.cwd() / "llama"
temp_onnx_dir.mkdir(exist_ok=True)
fp_onnx_path = temp_onnx_dir / "llama2_fp32.onnx"

generate_llama_fp_onnx(
    str(checkpoint_f_path),
    str(fp_onnx_path),
    model_params["sequence_len"],
    model_params["dim"],
    model_params["vocab_size"],
    model_params["n_layers"],
    model_params["n_heads"],
)

==========================================================================================
Layer (type:depth-idx)                   Output Shape              Param #
==========================================================================================
Transformer                              [1, 1, 32000]             --
├─Embedding: 1-1                         [1, 256, 288]             9,216,000
├─Dropout: 1-2                           [1, 256, 288]             --
├─ModuleList: 1-3                        --                        --
│    └─TransformerBlock: 2-1             [1, 256, 288]             --
│    │    └─RMSNorm: 3-1                 [1, 256, 288]             288
│    │    └─Attention: 3-2               --                        331,776
│    │    └─RMSNorm: 3-3                 [1, 256, 288]             288
│    │    └─FeedForward: 3-4             --                        663,552
│    └─TransformerBlock: 2-2             [1, 256, 288]             --
│    │    └─RMSNorm: 3-5                 [1, 256, 288]             288
│    │    └─Attention: 3-6               --                        331,776
│    │    └─RMSNorm: 3-7                 [1, 256, 288]             288
│    │    └─FeedForward: 3-8             --                        663,552
│    └─TransformerBlock: 2-3             [1, 256, 288]             --
│    │    └─RMSNorm: 3-9                 [1, 256, 288]             288
│    │    └─Attention: 3-10              --                        331,776
│    │    └─RMSNorm: 3-11                [1, 256, 288]             288
│    │    └─FeedForward: 3-12            --                        663,552
│    └─TransformerBlock: 2-4             [1, 256, 288]             --
│    │    └─RMSNorm: 3-13                [1, 256, 288]             288
│    │    └─Attention: 3-14              --                        331,776
│    │    └─RMSNorm: 3-15                [1, 256, 288]             288
│    │    └─FeedForward: 3-16            --                        663,552
│    └─TransformerBlock: 2-5             [1, 256, 288]             --
│    │    └─RMSNorm: 3-17                [1, 256, 288]             288
│    │    └─Attention: 3-18              --                        331,776
│    │    └─RMSNorm: 3-19                [1, 256, 288]             288
│    │    └─FeedForward: 3-20            --                        663,552
│    └─TransformerBlock: 2-6             [1, 256, 288]             --
│    │    └─RMSNorm: 3-21                [1, 256, 288]             288
│    │    └─Attention: 3-22              --                        331,776
│    │    └─RMSNorm: 3-23                [1, 256, 288]             288
│    │    └─FeedForward: 3-24            --                        663,552
├─RMSNorm: 1-4                           [1, 256, 288]             288
├─Linear: 1-5                            [1, 1, 32000]             9,216,000
==========================================================================================
Total params: 24,407,712
Trainable params: 24,407,712
Non-trainable params: 0
Total mult-adds (M): 24.41
==========================================================================================
Input size (MB): 0.00
Forward/backward pass size (MB): 45.08
Params size (MB): 97.63
Estimated Total Size (MB): 142.72
==========================================================================================


/quadric/sdk-cli/examples/models/llama/model/model.py:63: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
  assert freqs_cis.shape == (


Saved model to /quadric/sdk-cli/examples/models/llama/llama/llama2_fp32.onnx

Download datasets and Generate quantized ONNX graph

from llama_quantizer import quantize_llama_onnx

quant_onnx_path = temp_onnx_dir / "llama2_optimized_sym_int8_q.onnx"
dataset_path = Path.cwd() / "data"

quantize_llama_onnx(model_params, fp_onnx_path, quant_onnx_path, dataset_path, temp_onnx_dir)

gc.collect()

Compressed dataset file not found at the given path... (/quadric/sdk-cli/examples/models/llama/data/TinyStories_all_data.tar.gz)
Downloading dataset...
Unpacking /quadric/sdk-cli/examples/models/llama/data/TinyStories_all_data.tar.gz...


Using static quantization schema (dataset: json, method: CalibrationMethod.MinMax)
Creating calibrator: CalibrationMethod.MinMax(CalibrationConfig(dataset_name='json', dataset_config_name='default', dataset_split='train', dataset_num_samples=5000, method=<CalibrationMethod.MinMax: 0>, num_bins=None, num_quantized_bins=None, percentile=None, moving_average=False, averaging_constant=0.01))
Collecting tensors statistics...
Computing calibration ranges
Creating static quantizer: QOperator (mode: QLinearOps, schema: s8/s8, channel-wise: False)
Quantizing model...
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.0/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.1/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.2/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.3/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.4/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Unsqueeze_18_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_9_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Gather_8_output_0', type is tensor_type {
  elem_type: 7
  shape {
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_1_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/layers.5/Slice_3_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 1
    }
  }
}
.
Saving quantized model at: /quadric/sdk-cli/examples/models/llama/llama (external data format: False)
Configuration saved in /quadric/sdk-cli/examples/models/llama/llama/ort_config.json


Quantized graph with fixed input shape is saved at /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
Cleaning up temp files...
Done.





4193

Generate Custom-ops replaced ONNX graph

from custom_op_replacer import prepare_custom_ops_graph

custom_op_onnx_path = temp_onnx_dir / "llama2_custom_op_int8_q.onnx"

prepare_custom_ops_graph(model_params, str(quant_onnx_path), str(custom_op_onnx_path))

gc.collect()

Found already existing dataset...





7978

Generate C++ code and compile.

from tvm.contrib.epu.chimera_job.chimera_job import ChimeraJob

cgc_job = ChimeraJob(
    model_p=str(custom_op_onnx_path),
    onnx_ort_override_p=str(quant_onnx_path),
    quiet_iss=False,
)
cgc_job.compile(quiet=True)

2026-06-19 04:12 - INFO - epu - chimera_job - Overriding onnx model ort execution with the onnx model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx

Define the number of parallel threads you can run and the total number of runs

num_threads = 6
total_runs = 24

Loading input data from TinyStories dataset

import numpy as np
from llama_utils import load_and_tokenize_validation_data

dataset_path = Path.cwd() / "data"

val_data_array = load_and_tokenize_validation_data(model_params, str(dataset_path), total_runs)

inp_list = []  # must be in int32 type
target_idx_list = []

for inp_toks, target_tok in val_data_array:
    inp_list.append({"input": inp_toks.astype(np.int32)})
    target_idx_list.append(target_tok)

Downloading validation data...
Download completed! (saved to /quadric/sdk-cli/examples/models/llama/data/TinyStories-valid.txt)

Run multiple instances of ISS and ORT in parallel threads

results_iss_list = cgc_job.run_batch_inference_harness(
    inputs=inp_list,
    threads=num_threads,
)
results_ort_list = cgc_job.run_batch_ort_harness(
    inputs=inp_list,
    threads=num_threads,
)

Processing: 100%|██████████████████████████████| 24/24 [02:53<00:00,  7.24s/it]
2026-06-19 04:17 - WARNING - epu - chimera_job - ORT is not threadsafe -- forcing single threaded batch execution
100%|██████████████████████████████████████████| 24/24 [00:03<00:00,  7.99it/s]

Calculate Top-1 and Top-5 accuracies

(Compared to ORT output of quantized ONNX)

top1_count = 0
top5_count = 0

for iss, ort in zip(results_iss_list, results_ort_list):
    ort_arg_1 = np.argmax(ort["output"].flatten())
    iss_arg_1 = np.argmax(iss["output"].flatten())
    iss_arg_5 = np.argsort(iss["output"].flatten())[::-1][:5]
    if ort_arg_1 == iss_arg_1:
        top1_count = top1_count + 1
    if ort_arg_1 in iss_arg_5:
        top5_count = top5_count + 1

top1_acc = 100 * top1_count / len(results_iss_list)
top5_acc = 100 * top5_count / len(results_iss_list)

print("Accuracy of ISS run. (vs quantized ONNX run)")
print(f"Top-1 Accuracy : {top1_acc:>7.3f}%    ({top1_count}/{len(results_iss_list)})")
print(f"Top-5 Accuracy : {top5_acc:>7.3f}%    ({top5_count}/{len(results_iss_list)})")

Accuracy of ISS run. (vs quantized ONNX run)
Top-1 Accuracy :  87.500%    (21/24)
Top-5 Accuracy :  95.833%    (23/24)

Perplexity scores.

import scipy

ort_loss = 0.0
iss_loss = 0.0
for target, iss, ort in zip(target_idx_list, results_iss_list, results_ort_list):
    sm_ort = scipy.special.softmax(ort["output"].flatten(), axis=-1)
    sm_iss = scipy.special.softmax(iss["output"].flatten(), axis=-1)
    ort_loss = ort_loss + (-1 * np.log(sm_ort[target]))
    iss_loss = iss_loss + (-1 * np.log(sm_iss[target]))

ort_loss = ort_loss / total_runs
iss_loss = iss_loss / total_runs

ppl_ort = np.exp(ort_loss)
ppl_iss = np.exp(iss_loss)

print("Perplexity Scores. (Lower is better)")
print("Perplexity of ORT run :", ppl_ort)
print("Perplexity of ISS run :", ppl_iss)

Perplexity Scores. (Lower is better)
Perplexity of ORT run : 3.6292876342732123
Perplexity of ISS run : 3.7721909737608446

Plot the correlation for each run

Set plot_correlations = True if required...

plot_correlations = False

if plot_correlations:
    import matplotlib.pyplot as plt
    import math

    plt_cols = min(5, total_runs)
    plt_rows = math.ceil(total_runs / plt_cols)
    fig, ax = plt.subplots(plt_rows, plt_cols, figsize=(20, 3 * plt_rows))
    ax = ax.reshape((plt_rows, plt_cols))

    i = 0
    while i < plt_cols * plt_rows:
        plt_c = i % plt_cols
        plt_r = i // plt_cols
        if i < len(results_iss_list):
            ax[plt_r, plt_c].plot(
                results_ort_list[i]["output"].flatten(),
                results_iss_list[i]["output"].flatten(),
                ".",
            )
            ax[plt_r, plt_c].get_xaxis().set_visible(False)
            ax[plt_r, plt_c].get_yaxis().set_visible(False)
        else:
            ax[plt_r, plt_c].set_axis_off()

        i = i + 1

    plt.show()

Save ISS and ORT output tensors

Set dump_data = True if required...

dump_data = False
if dump_data:
    dump_data_dir = Path.cwd() / "data_dumps"
    dump_data_dir.mkdir(exist_ok=True)
    iss = np.stack([results_iss_list[i]["output"].flatten() for i in range(len(results_ort_list))])
    ort = np.stack([results_ort_list[i]["output"].flatten() for i in range(len(results_iss_list))])
    np.save(f"{dump_data_dir}/iss_out_batch.npy", iss)
    np.save(f"{dump_data_dir}/ort_out_batch.npy", ort)

Text Generation Example

We demonstrate the quantized model running autoregressively and producing stories on ORT and ISS.

Load Vocabulary

from llama_run import load_vocab
import numpy as np

vocab_size = 32000
filename = "model/tokenizer.bin"
vocab = load_vocab(vocab_size, filename)
idx = np.array([[1]])  # Starting token

Quantized Model on ORT

from llama_run import generate_tokens_onnx
import onnxruntime

sess_opt = onnxruntime.SessionOptions()
sess_opt.intra_op_num_threads = 1
session = onnxruntime.InferenceSession(quant_onnx_path, sess_opt)

generated_sequence = generate_tokens_onnx(
    session,
    idx,
    max_new_tokens=256,
    vocab=vocab,
    do_padding=True,
    temperature=0.0,
    padding_token=1,
    max_seq_len=256,
)

<s> Once upon a time, there was a little girl named Jane was in the forest. She was a very special, but she was a bit scared of the dark and the night when the sun went on the sticks and a night was in the dark. She was scared, but she was also very scared. She was a brave and strong and she could see a light at night.
"Are you a ghost?" Jane, who was her best friend, who had just been looking for her. She had a bright, shiny stone and a big smile on her face.
"Don't be scared, I'm not so scary," she said. "I'm just danc that the sun is pointing at the moon and the stars. It's so beautiful!"
The two of them started to dance and sing and danced around the room. Jane was so happy and she clapped her hands and shouted, "I love you, Jane!"
"I love it, Jane, I'm so glad you're here," she said. "You're the best friend ever, I'm so happy to be here."
Jane and Tom hugged her and said goodbye.
Tokens per second: 9.359761089134937

Quantized Model on ISS

from llama_run import generate_tokens_iss

generated_sequence = generate_tokens_iss(
    cgc_job,
    idx,
    max_new_tokens=10,
    vocab=vocab,
    do_padding=True,
    temperature=0.0,
    padding_token=1,
    max_seq_len=256,
)

<s>

2026-06-19 04:18 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x70721189c0d0>
2026-06-19 04:18 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x707047cbf520>
2026-06-19 04:18 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:18 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:18 - INFO - epu - iss_testing - Done 0:00:00.129613


 Once

2026-06-19 04:18 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x70736c3b2830>
2026-06-19 04:19 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c92ca9b0>
2026-06-19 04:19 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:19 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:19 - INFO - epu - iss_testing - Done 0:00:00.131211


 upon

2026-06-19 04:19 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c92a5390>
2026-06-19 04:20 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x70736c3cdc30>
2026-06-19 04:20 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:20 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:20 - INFO - epu - iss_testing - Done 0:00:00.120595


 a

2026-06-19 04:20 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c12da6e0>
2026-06-19 04:20 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c037c640>
2026-06-19 04:20 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:20 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:20 - INFO - epu - iss_testing - Done 0:00:00.116116


 time

2026-06-19 04:20 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x70736c3cdc30>
2026-06-19 04:21 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c03116c0>
2026-06-19 04:21 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:21 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:21 - INFO - epu - iss_testing - Done 0:00:00.131953


,

2026-06-19 04:21 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c12db4f0>
2026-06-19 04:22 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071b8129a50>
2026-06-19 04:22 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:22 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:22 - INFO - epu - iss_testing - Done 0:00:00.131809


 there

2026-06-19 04:22 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x70736c3cdc30>
2026-06-19 04:22 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c92b38e0>
2026-06-19 04:22 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:22 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:22 - INFO - epu - iss_testing - Done 0:00:00.129971


 was

2026-06-19 04:22 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c12119f0>
2026-06-19 04:23 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c1212050>
2026-06-19 04:23 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:23 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:23 - INFO - epu - iss_testing - Done 0:00:00.115088


 a

2026-06-19 04:23 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071b8128490>
2026-06-19 04:24 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071b81287c0>
2026-06-19 04:24 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:24 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:24 - INFO - epu - iss_testing - Done 0:00:00.128110


 little

2026-06-19 04:24 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c923dc30>
2026-06-19 04:24 - INFO - epu - iss_testing - No tranges found for input, use default float range: <tvm.contrib.epu.interval.Interval object at 0x7071c923c640>
2026-06-19 04:24 - INFO - epu - iss_testing - Started Executing Onnxruntime...
2026-06-19 04:24 - INFO - epu - chimera_job - running ort reference with overriden model: /quadric/sdk-cli/examples/models/llama/llama/llama2_optimized_sym_int8_q.onnx
2026-06-19 04:24 - INFO - epu - iss_testing - Done 0:00:00.132809


 girl
Full generated sequence: <s> Once upon a time, there was a little girl

(Option) Quantizing Llama-15M with Smooth Quantization

Smooth quantization is an accurate and efficient post-training quantization (PTQ) solution for LLMs. Normally, activations in ML models are much harder to quantize than weights due to the presence of outliers. However, different tokens exhibit similar variations across their channels. Based on this observation, SmoothQuant offline migrates the quantization difficulty from activations to weight.

This notebook is for quantizing Llama-15M model using the smooth quantization method.

Further details about smooth quantization can be found in the paper.

• Guangxuan Xiao, Ji Lin, Mickael Seznec, Hao Wu, Julien Demouth, Song Han. SmoothQuant: Accurate and Efficient Post-Training Quantization for Large Language Models

How to Run the Option

In order to run the following, please do the following

• Set True to the following RUN_OPTION.

RUN_OPTION = False

Define ModelHelpers

import numpy as np
from sdk_cli.utils import model_helpers
from model.tokenizer import Tokenizer

if RUN_OPTION:
    tokenizer = Tokenizer()

    def tokenize_val(text):
        text = text.strip()  # get rid of leading/trailing whitespace
        tokens = tokenizer.encode(text, bos=True, eos=False)  # encode the text, use BOS
        np_tokens = np.array(tokens)
        return np_tokens.flatten()

    mh_val = model_helpers.TextModelHelper(tokenize_val, model_params["sequence_len"])

Download datasets and Generate smooth quantized ONNX graph

from llama_smooth_quantization import llama_smooth_quantization

if RUN_OPTION:
    model_size = "15M"
    model_params = supported_models_dict[model_size]

    fp_onnx_path = temp_onnx_dir / "llama2_fp32.onnx"
    smooth_quant_alpha = 0.5
    quant_onnx_path = temp_onnx_dir / f"llama2_opt_smooth_quant_{smooth_quant_alpha}.onnx"
    dataset_path = Path.cwd() / "data_smq"
    temp_onnx_dir = Path.cwd() / "llama"

    llama_smooth_quantization(
        model_params,
        fp_onnx_path,
        quant_onnx_path,
        smooth_quant_alpha,
        dataset_path,
        temp_onnx_dir,
    )

Run multiple ORT instances in parallel threads

This will run the inputs through both quantized and float32 ONNX graphs...

import multiprocess
import onnxruntime
from tqdm import tqdm


def run_ort_inference(zipped_params):
    (fp_onnx_path, q_onnx_path, inp) = zipped_params

    def single_ort_inference(onnx_path, inp):
        sess_model = onnxruntime.InferenceSession(onnx_path)
        onnx_out = sess_model.run([], {"input": inp})[0].flatten()
        return onnx_out

    fp_out = single_ort_inference(fp_onnx_path, inp)
    q_out = single_ort_inference(q_onnx_path, inp)

    return (fp_out, q_out)


if RUN_OPTION:
    total_runs = 1000
    num_threads = 3

    val_story_list = mh_val.get_data_list("data_smq/TinyStories-valid.txt")
    val_data = mh_val.get_batch_data(val_story_list)
    val_d_list = val_data[:total_runs]
    total_runs = len(val_d_list)

    results_fp_list = []
    results_q_list = []

    with multiprocess.Pool(num_threads) as p:
        zipped_inp = zip([fp_onnx_path] * total_runs, [quant_onnx_path] * total_runs, val_d_list)
        results = list(tqdm(p.imap(run_ort_inference, zipped_inp), total=total_runs))

    for fp_out, q_out in results:
        results_fp_list.append(fp_out)
        results_q_list.append(q_out)

Calculate Top-1 and Top-5 accuracies of quantized ONNX graph

(Compared to output of float32 ONNX graph)

if RUN_OPTION:
    top1_count = 0
    top5_count = 0

    for q_out, fp_out in zip(results_q_list, results_fp_list):
        fp_arg_1 = np.argmax(fp_out.flatten())
        q_arg_1 = np.argmax(q_out.flatten())
        q_arg_5 = np.argsort(q_out.flatten())[::-1][:5]
        if fp_arg_1 == q_arg_1:
            top1_count = top1_count + 1
        if fp_arg_1 in q_arg_5:
            top5_count = top5_count + 1

    top1_acc = 100 * top1_count / len(results_fp_list)
    top5_acc = 100 * top5_count / len(results_fp_list)

    print(f"Accuracy of Quantized run. (vs FP ONNX run) --- SmoothQuantAlpha: {smooth_quant_alpha}")
    print(f"Top-1 Accuracy : {top1_acc:>7.3f}%    ({top1_count}/{len(results_fp_list)})")
    print(f"Top-5 Accuracy : {top5_acc:>7.3f}%    ({top5_count}/{len(results_fp_list)})")
    print()