Quantization of Image Classification Models¶
This tutorial demostrates how to apply INT8 quantization to Image Classification model using Post-training Optimization Tool API. The Mobilenet V2 model trained on Cifar10 dataset is used as an example. The code of this tutorial is designed to be extandable to custom model and dataset. It is assumed that OpenVINO is already installed. This tutorial consists of the following steps: - Prepare the model for quantization - Define data loading and accuracy validation functionality - Run optimization pipeline - Compare accuracy of the original and quantized models - Compare performance of the original and quantized models - Compare results on one picture
import os
from pathlib import Path
import matplotlib.pyplot as plt
import numpy as np
import torch
from addict import Dict
from compression.api import DataLoader, Metric
from compression.engines.ie_engine import IEEngine
from compression.graph import load_model, save_model
from compression.graph.model_utils import compress_model_weights
from compression.pipeline.initializer import create_pipeline
from openvino.runtime import Core
from torchvision import transforms
from torchvision.datasets import CIFAR10
# Set the data and model directories
DATA_DIR = 'data'
MODEL_DIR = 'model'
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(MODEL_DIR, exist_ok=True)
Prepare the Model¶
Model preparation stage has the following steps: - Download PyTorch model from Torchvision repository - Convert it to ONNX format - Run OpenVINO Model Optimizer tool to convert ONNX to OpenVINO Intermediate Representation (IR)
model = torch.hub.load("chenyaofo/pytorch-cifar-models", "cifar10_mobilenetv2_x1_0", pretrained=True)
model.eval()
dummy_input = torch.randn(1, 3, 32, 32)
onnx_model_path = Path(MODEL_DIR) / 'mobilenet_v2.onnx'
ir_model_xml = onnx_model_path.with_suffix('.xml')
ir_model_bin = onnx_model_path.with_suffix('.bin')
torch.onnx.export(model, dummy_input, onnx_model_path, verbose=True)
# Run OpenVINO Model Optimization tool to convert ONNX to OpenVINO IR
!mo --framework=onnx --data_type=FP16 --input_shape=[1,3,32,32] -m $onnx_model_path --output_dir $MODEL_DIR
Downloading: "https://github.com/chenyaofo/pytorch-cifar-models/archive/master.zip" to /home/runner/.cache/torch/hub/master.zip
Downloading: "https://github.com/chenyaofo/pytorch-cifar-models/releases/download/mobilenetv2/cifar10_mobilenetv2_x1_0-fe6a5b48.pt" to /home/runner/.cache/torch/hub/checkpoints/cifar10_mobilenetv2_x1_0-fe6a5b48.pt
0%| | 0.00/8.77M [00:00<?, ?B/s]
graph(%input.1 : Float(1:3072, 3:1024, 32:32, 32:1, requires_grad=0, device=cpu),
%classifier.1.weight : Float(10:1280, 1280:1, requires_grad=1, device=cpu),
%classifier.1.bias : Float(10:1, requires_grad=1, device=cpu),
%468 : Float(32:27, 3:9, 3:3, 3:1, requires_grad=0, device=cpu),
%469 : Float(32:1, requires_grad=0, device=cpu),
%471 : Float(32:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%472 : Float(32:1, requires_grad=0, device=cpu),
%474 : Float(16:32, 32:1, 1:1, 1:1, requires_grad=0, device=cpu),
%475 : Float(16:1, requires_grad=0, device=cpu),
%477 : Float(96:16, 16:1, 1:1, 1:1, requires_grad=0, device=cpu),
%478 : Float(96:1, requires_grad=0, device=cpu),
%480 : Float(96:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%481 : Float(96:1, requires_grad=0, device=cpu),
%483 : Float(24:96, 96:1, 1:1, 1:1, requires_grad=0, device=cpu),
%484 : Float(24:1, requires_grad=0, device=cpu),
%486 : Float(144:24, 24:1, 1:1, 1:1, requires_grad=0, device=cpu),
%487 : Float(144:1, requires_grad=0, device=cpu),
%489 : Float(144:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%490 : Float(144:1, requires_grad=0, device=cpu),
%492 : Float(24:144, 144:1, 1:1, 1:1, requires_grad=0, device=cpu),
%493 : Float(24:1, requires_grad=0, device=cpu),
%495 : Float(144:24, 24:1, 1:1, 1:1, requires_grad=0, device=cpu),
%496 : Float(144:1, requires_grad=0, device=cpu),
%498 : Float(144:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%499 : Float(144:1, requires_grad=0, device=cpu),
%501 : Float(32:144, 144:1, 1:1, 1:1, requires_grad=0, device=cpu),
%502 : Float(32:1, requires_grad=0, device=cpu),
%504 : Float(192:32, 32:1, 1:1, 1:1, requires_grad=0, device=cpu),
%505 : Float(192:1, requires_grad=0, device=cpu),
%507 : Float(192:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%508 : Float(192:1, requires_grad=0, device=cpu),
%510 : Float(32:192, 192:1, 1:1, 1:1, requires_grad=0, device=cpu),
%511 : Float(32:1, requires_grad=0, device=cpu),
%513 : Float(192:32, 32:1, 1:1, 1:1, requires_grad=0, device=cpu),
%514 : Float(192:1, requires_grad=0, device=cpu),
%516 : Float(192:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%517 : Float(192:1, requires_grad=0, device=cpu),
%519 : Float(32:192, 192:1, 1:1, 1:1, requires_grad=0, device=cpu),
%520 : Float(32:1, requires_grad=0, device=cpu),
%522 : Float(192:32, 32:1, 1:1, 1:1, requires_grad=0, device=cpu),
%523 : Float(192:1, requires_grad=0, device=cpu),
%525 : Float(192:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%526 : Float(192:1, requires_grad=0, device=cpu),
%528 : Float(64:192, 192:1, 1:1, 1:1, requires_grad=0, device=cpu),
%529 : Float(64:1, requires_grad=0, device=cpu),
%531 : Float(384:64, 64:1, 1:1, 1:1, requires_grad=0, device=cpu),
%532 : Float(384:1, requires_grad=0, device=cpu),
%534 : Float(384:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%535 : Float(384:1, requires_grad=0, device=cpu),
%537 : Float(64:384, 384:1, 1:1, 1:1, requires_grad=0, device=cpu),
%538 : Float(64:1, requires_grad=0, device=cpu),
%540 : Float(384:64, 64:1, 1:1, 1:1, requires_grad=0, device=cpu),
%541 : Float(384:1, requires_grad=0, device=cpu),
%543 : Float(384:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%544 : Float(384:1, requires_grad=0, device=cpu),
%546 : Float(64:384, 384:1, 1:1, 1:1, requires_grad=0, device=cpu),
%547 : Float(64:1, requires_grad=0, device=cpu),
%549 : Float(384:64, 64:1, 1:1, 1:1, requires_grad=0, device=cpu),
%550 : Float(384:1, requires_grad=0, device=cpu),
%552 : Float(384:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%553 : Float(384:1, requires_grad=0, device=cpu),
%555 : Float(64:384, 384:1, 1:1, 1:1, requires_grad=0, device=cpu),
%556 : Float(64:1, requires_grad=0, device=cpu),
%558 : Float(384:64, 64:1, 1:1, 1:1, requires_grad=0, device=cpu),
%559 : Float(384:1, requires_grad=0, device=cpu),
%561 : Float(384:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%562 : Float(384:1, requires_grad=0, device=cpu),
%564 : Float(96:384, 384:1, 1:1, 1:1, requires_grad=0, device=cpu),
%565 : Float(96:1, requires_grad=0, device=cpu),
%567 : Float(576:96, 96:1, 1:1, 1:1, requires_grad=0, device=cpu),
%568 : Float(576:1, requires_grad=0, device=cpu),
%570 : Float(576:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%571 : Float(576:1, requires_grad=0, device=cpu),
%573 : Float(96:576, 576:1, 1:1, 1:1, requires_grad=0, device=cpu),
%574 : Float(96:1, requires_grad=0, device=cpu),
%576 : Float(576:96, 96:1, 1:1, 1:1, requires_grad=0, device=cpu),
%577 : Float(576:1, requires_grad=0, device=cpu),
%579 : Float(576:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%580 : Float(576:1, requires_grad=0, device=cpu),
%582 : Float(96:576, 576:1, 1:1, 1:1, requires_grad=0, device=cpu),
%583 : Float(96:1, requires_grad=0, device=cpu),
%585 : Float(576:96, 96:1, 1:1, 1:1, requires_grad=0, device=cpu),
%586 : Float(576:1, requires_grad=0, device=cpu),
%588 : Float(576:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%589 : Float(576:1, requires_grad=0, device=cpu),
%591 : Float(160:576, 576:1, 1:1, 1:1, requires_grad=0, device=cpu),
%592 : Float(160:1, requires_grad=0, device=cpu),
%594 : Float(960:160, 160:1, 1:1, 1:1, requires_grad=0, device=cpu),
%595 : Float(960:1, requires_grad=0, device=cpu),
%597 : Float(960:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%598 : Float(960:1, requires_grad=0, device=cpu),
%600 : Float(160:960, 960:1, 1:1, 1:1, requires_grad=0, device=cpu),
%601 : Float(160:1, requires_grad=0, device=cpu),
%603 : Float(960:160, 160:1, 1:1, 1:1, requires_grad=0, device=cpu),
%604 : Float(960:1, requires_grad=0, device=cpu),
%606 : Float(960:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%607 : Float(960:1, requires_grad=0, device=cpu),
%609 : Float(160:960, 960:1, 1:1, 1:1, requires_grad=0, device=cpu),
%610 : Float(160:1, requires_grad=0, device=cpu),
%612 : Float(960:160, 160:1, 1:1, 1:1, requires_grad=0, device=cpu),
%613 : Float(960:1, requires_grad=0, device=cpu),
%615 : Float(960:9, 1:9, 3:3, 3:1, requires_grad=0, device=cpu),
%616 : Float(960:1, requires_grad=0, device=cpu),
%618 : Float(320:960, 960:1, 1:1, 1:1, requires_grad=0, device=cpu),
%619 : Float(320:1, requires_grad=0, device=cpu),
%621 : Float(1280:320, 320:1, 1:1, 1:1, requires_grad=0, device=cpu),
%622 : Float(1280:1, requires_grad=0, device=cpu)):
%467 : Float(1:32768, 32:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%input.1, %468, %469)
%317 : Float(1:32768, 32:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%467) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%470 : Float(1:32768, 32:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=32, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%317, %471, %472)
%320 : Float(1:32768, 32:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%470) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%473 : Float(1:16384, 16:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%320, %474, %475)
%476 : Float(1:98304, 96:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%473, %477, %478)
%325 : Float(1:98304, 96:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%476) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%479 : Float(1:98304, 96:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=96, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%325, %480, %481)
%328 : Float(1:98304, 96:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%479) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%482 : Float(1:24576, 24:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%328, %483, %484)
%485 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%482, %486, %487)
%333 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%485) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%488 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=144, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%333, %489, %490)
%336 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%488) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%491 : Float(1:24576, 24:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%336, %492, %493)
%339 : Float(1:24576, 24:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Add(%482, %491) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%494 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%339, %495, %496)
%342 : Float(1:147456, 144:1024, 32:32, 32:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%494) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%497 : Float(1:36864, 144:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=144, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[2, 2]](%342, %498, %499)
%345 : Float(1:36864, 144:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%497) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%500 : Float(1:8192, 32:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%345, %501, %502)
%503 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%500, %504, %505)
%350 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%503) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%506 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=192, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%350, %507, %508)
%353 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%506) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%509 : Float(1:8192, 32:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%353, %510, %511)
%356 : Float(1:8192, 32:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Add(%500, %509) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%512 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%356, %513, %514)
%359 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%512) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%515 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=192, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%359, %516, %517)
%362 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%515) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%518 : Float(1:8192, 32:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%362, %519, %520)
%365 : Float(1:8192, 32:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Add(%356, %518) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%521 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%365, %522, %523)
%368 : Float(1:49152, 192:256, 16:16, 16:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%521) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%524 : Float(1:12288, 192:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=192, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[2, 2]](%368, %525, %526)
%371 : Float(1:12288, 192:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%524) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%527 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%371, %528, %529)
%530 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%527, %531, %532)
%376 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%530) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%533 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=384, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%376, %534, %535)
%379 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%533) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%536 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%379, %537, %538)
%382 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Add(%527, %536) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%539 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%382, %540, %541)
%385 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%539) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%542 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=384, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%385, %543, %544)
%388 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%542) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%545 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%388, %546, %547)
%391 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Add(%382, %545) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%548 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%391, %549, %550)
%394 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%548) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%551 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=384, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%394, %552, %553)
%397 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%551) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%554 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%397, %555, %556)
%400 : Float(1:4096, 64:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Add(%391, %554) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%557 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%400, %558, %559)
%403 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%557) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%560 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=384, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%403, %561, %562)
%406 : Float(1:24576, 384:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%560) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%563 : Float(1:6144, 96:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%406, %564, %565)
%566 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%563, %567, %568)
%411 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%566) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%569 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=576, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%411, %570, %571)
%414 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%569) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%572 : Float(1:6144, 96:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%414, %573, %574)
%417 : Float(1:6144, 96:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Add(%563, %572) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%575 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%417, %576, %577)
%420 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%575) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%578 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=576, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%420, %579, %580)
%423 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%578) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%581 : Float(1:6144, 96:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%423, %582, %583)
%426 : Float(1:6144, 96:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Add(%417, %581) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%584 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%426, %585, %586)
%429 : Float(1:36864, 576:64, 8:8, 8:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%584) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%587 : Float(1:9216, 576:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=576, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[2, 2]](%429, %588, %589)
%432 : Float(1:9216, 576:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%587) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%590 : Float(1:2560, 160:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%432, %591, %592)
%593 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%590, %594, %595)
%437 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%593) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%596 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=960, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%437, %597, %598)
%440 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%596) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%599 : Float(1:2560, 160:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%440, %600, %601)
%443 : Float(1:2560, 160:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Add(%590, %599) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%602 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%443, %603, %604)
%446 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%602) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%605 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=960, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%446, %606, %607)
%449 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%605) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%608 : Float(1:2560, 160:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%449, %609, %610)
%452 : Float(1:2560, 160:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Add(%443, %608) # /home/runner/.cache/torch/hub/chenyaofo_pytorch-cifar-models_master/pytorch_cifar_models/mobilenetv2.py:144:0
%611 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%452, %612, %613)
%455 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%611) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%614 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=960, kernel_shape=[3, 3], pads=[1, 1, 1, 1], strides=[1, 1]](%455, %615, %616)
%458 : Float(1:15360, 960:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%614) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%617 : Float(1:5120, 320:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%458, %618, %619)
%620 : Float(1:20480, 1280:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[1, 1], pads=[0, 0, 0, 0], strides=[1, 1]](%617, %621, %622)
%463 : Float(1:20480, 1280:16, 4:4, 4:1, requires_grad=1, device=cpu) = onnx::Clip[max=6., min=0.](%620) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1186:0
%464 : Float(1:1280, 1280:1, 1:1, 1:1, requires_grad=1, device=cpu) = onnx::GlobalAveragePool(%463) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:936:0
%465 : Float(1:1280, 1280:1, requires_grad=1, device=cpu) = onnx::Flatten[axis=1](%464) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:983:0
%466 : Float(1:10, 10:1, requires_grad=1, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%465, %classifier.1.weight, %classifier.1.bias) # /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/torch/nn/functional.py:1690:0
return (%466)
Model Optimizer arguments:
Common parameters:
- Path to the Input Model: /home/runner/work/openvino_notebooks/openvino_notebooks/notebooks/113-image-classification-quantization/model/mobilenet_v2.onnx
- Path for generated IR: /home/runner/work/openvino_notebooks/openvino_notebooks/notebooks/113-image-classification-quantization/model
- IR output name: mobilenet_v2
- Log level: ERROR
- Batch: Not specified, inherited from the model
- Input layers: Not specified, inherited from the model
- Output layers: Not specified, inherited from the model
- Input shapes: [1,3,32,32]
- Source layout: Not specified
- Target layout: Not specified
- Layout: Not specified
- Mean values: Not specified
- Scale values: Not specified
- Scale factor: Not specified
- Precision of IR: FP16
- Enable fusing: True
- User transformations: Not specified
- Reverse input channels: False
- Enable IR generation for fixed input shape: False
- Use the transformations config file: None
Advanced parameters:
- Force the usage of legacy Frontend of Model Optimizer for model conversion into IR: False
- Force the usage of new Frontend of Model Optimizer for model conversion into IR: False
OpenVINO runtime found in: /opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/site-packages/openvino
OpenVINO runtime version: 2022.1.0-7019-cdb9bec7210-releases/2022/1
Model Optimizer version: 2022.1.0-7019-cdb9bec7210-releases/2022/1
[ SUCCESS ] Generated IR version 11 model.
[ SUCCESS ] XML file: /home/runner/work/openvino_notebooks/openvino_notebooks/notebooks/113-image-classification-quantization/model/mobilenet_v2.xml
[ SUCCESS ] BIN file: /home/runner/work/openvino_notebooks/openvino_notebooks/notebooks/113-image-classification-quantization/model/mobilenet_v2.bin
[ SUCCESS ] Total execution time: 0.69 seconds.
[ SUCCESS ] Memory consumed: 86 MB.
It's been a while, check for a new version of Intel(R) Distribution of OpenVINO(TM) toolkit here https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit/download.html?cid=other&source=prod&campid=ww_2022_bu_IOTG_OpenVINO-2022-1&content=upg_all&medium=organic or on the GitHub*
[ INFO ] The model was converted to IR v11, the latest model format that corresponds to the source DL framework input/output format. While IR v11 is backwards compatible with OpenVINO Inference Engine API v1.0, please use API v2.0 (as of 2022.1) to take advantage of the latest improvements in IR v11.
Find more information about API v2.0 and IR v11 at https://docs.openvino.ai
Define Data Loader¶
At this step the DataLoader interface from POT API is implemented.
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))])
dataset = CIFAR10(root=DATA_DIR, train=False, transform=transform, download=True)
Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to data/cifar-10-python.tar.gz
0it [00:00, ?it/s]
Extracting data/cifar-10-python.tar.gz to data
# create DataLoader from CIFAR10 dataset
class CifarDataLoader(DataLoader):
def __init__(self, config):
"""
Initialize config and dataset.
:param config: created config with DATA_DIR path.
"""
if not isinstance(config, Dict):
config = Dict(config)
super().__init__(config)
self.indexes, self.pictures, self.labels = self.load_data(dataset)
def __len__(self):
return len(self.labels)
def __getitem__(self, index):
"""
Return one sample of index, label and picture.
:param index: index of the taken sample.
"""
if index >= len(self):
raise IndexError
return (self.indexes[index], self.labels[index]), self.pictures[index].numpy()
def load_data(self, dataset):
"""
Load dataset in needed format.
:param dataset: downloaded dataset.
"""
pictures, labels, indexes = [], [], []
for idx, sample in enumerate(dataset):
pictures.append(sample[0])
labels.append(sample[1])
indexes.append(idx)
return indexes, pictures, labels
Define Accuracy Metric Calculation¶
At this step the Metric interface for accuracy Top-1 metric is
implemented. It is used for validating accuracy of quantized model.
# Custom implementation of classification accuracy metric.
class Accuracy(Metric):
# Required methods
def __init__(self, top_k=1):
super().__init__()
self._top_k = top_k
self._name = 'accuracy@top{}'.format(self._top_k)
self._matches = []
@property
def value(self):
""" Returns accuracy metric value for the last model output. """
return {self._name: self._matches[-1]}
@property
def avg_value(self):
""" Returns accuracy metric value for all model outputs. """
return {self._name: np.ravel(self._matches).mean()}
def update(self, output, target):
""" Updates prediction matches.
:param output: model output
:param target: annotations
"""
if len(output) > 1:
raise Exception('The accuracy metric cannot be calculated '
'for a model with multiple outputs')
if isinstance(target, dict):
target = list(target.values())
predictions = np.argsort(output[0], axis=1)[:, -self._top_k:]
match = [float(t in predictions[i]) for i, t in enumerate(target)]
self._matches.append(match)
def reset(self):
""" Resets collected matches """
self._matches = []
def get_attributes(self):
"""
Returns a dictionary of metric attributes {metric_name: {attribute_name: value}}.
Required attributes: 'direction': 'higher-better' or 'higher-worse'
'type': metric type
"""
return {self._name: {'direction': 'higher-better',
'type': 'accuracy'}}
Run Quantization Pipeline and compare the accuracy of the original and quantized models¶
Here we define a configuration for our quantization pipeline and run it.
NOTE: we use built-in IEEngine implementation of the Engine
interface from the POT API for model inference. IEEngine is built on
top of OpenVINO Python* API for inference and provides basic
functionality for inference of simple models. If you have a more
complicated inference flow for your model/models you should create your
own implementation of Engine interface, for example by inheriting
from IEEngine and extending it.
model_config = Dict({
'model_name': 'mobilenet_v2',
'model': ir_model_xml,
'weights': ir_model_bin
})
engine_config = Dict({
'device': 'CPU',
'stat_requests_number': 2,
'eval_requests_number': 2
})
dataset_config = {
'data_source': DATA_DIR
}
algorithms = [
{
'name': 'DefaultQuantization',
'params': {
'target_device': 'CPU',
'preset': 'performance',
'stat_subset_size': 300
}
}
]
# Steps 1-7: Model optimization
# Step 1: Load the model.
model = load_model(model_config)
# Step 2: Initialize the data loader.
data_loader = CifarDataLoader(dataset_config)
# Step 3 (Optional. Required for AccuracyAwareQuantization): Initialize the metric.
metric = Accuracy(top_k=1)
# Step 4: Initialize the engine for metric calculation and statistics collection.
engine = IEEngine(engine_config, data_loader, metric)
# Step 5: Create a pipeline of compression algorithms.
pipeline = create_pipeline(algorithms, engine)
# Step 6: Execute the pipeline.
compressed_model = pipeline.run(model)
# Step 7 (Optional): Compress model weights quantized precision
# in order to reduce the size of final .bin file.
compress_model_weights(compressed_model)
# Step 8: Save the compressed model to the desired path.
compressed_model_paths = save_model(model=compressed_model, save_path=MODEL_DIR, model_name="quantized_mobilenet_v2"
)
compressed_model_xml = compressed_model_paths[0]["model"]
compressed_model_bin = Path(compressed_model_paths[0]["model"]).with_suffix(".bin")
# Step 9: Compare accuracy of the original and quantized models.
metric_results = pipeline.evaluate(model)
if metric_results:
for name, value in metric_results.items():
print(f"Accuracy of the original model: {name}: {value}")
metric_results = pipeline.evaluate(compressed_model)
if metric_results:
for name, value in metric_results.items():
print(f"Accuracy of the optimized model: {name}: {value}")
Accuracy of the original model: accuracy@top1: 0.9341
Accuracy of the optimized model: accuracy@top1: 0.9341
Compare Performance of the Original and Quantized Models¶
Finally, we will measure the inference performance of the FP32 and INT8 models. To do this, we use Benchmark Tool - OpenVINO’s inference performance measurement tool.
NOTE: For more accurate performance, we recommended running benchmark_app in a terminal/command prompt after closing other applications. Run benchmark_app -m model.xml -d CPU to benchmark async inference on CPU for one minute. Change CPU to GPU to benchmark on GPU. Run benchmark_app –help to see an overview of all command line options.
# Inference FP16 model (IR)
!benchmark_app -m $ir_model_xml -d CPU -api async
[Step 1/11] Parsing and validating input arguments
[ WARNING ] -nstreams default value is determined automatically for a device. Although the automatic selection usually provides a reasonable performance, but it still may be non-optimal for some cases, for more information look at README.
[Step 2/11] Loading OpenVINO
[ WARNING ] PerformanceMode was not explicitly specified in command line. Device CPU performance hint will be set to THROUGHPUT.
[ INFO ] OpenVINO:
API version............. 2022.1.0-7019-cdb9bec7210-releases/2022/1
[ INFO ] Device info
CPU
openvino_intel_cpu_plugin version 2022.1
Build................... 2022.1.0-7019-cdb9bec7210-releases/2022/1
[Step 3/11] Setting device configuration
[ WARNING ] -nstreams default value is determined automatically for CPU device. Although the automatic selection usually provides a reasonable performance, but it still may be non-optimal for some cases, for more information look at README.
[Step 4/11] Reading network files
[ INFO ] Read model took 17.07 ms
[Step 5/11] Resizing network to match image sizes and given batch
[ INFO ] Network batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model input 'input.1' precision u8, dimensions ([N,C,H,W]): 1 3 32 32
[ INFO ] Model output '466' precision f32, dimensions ([...]): 1 10
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 90.71 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] DEVICE: CPU
[ INFO ] AVAILABLE_DEVICES , ['']
[ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS , (1, 1, 1)
[ INFO ] RANGE_FOR_STREAMS , (1, 2)
[ INFO ] FULL_DEVICE_NAME , Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
[ INFO ] OPTIMIZATION_CAPABILITIES , ['WINOGRAD', 'FP32', 'FP16', 'INT8', 'BIN', 'EXPORT_IMPORT']
[ INFO ] CACHE_DIR ,
[ INFO ] NUM_STREAMS , 1
[ INFO ] INFERENCE_NUM_THREADS , 0
[ INFO ] PERF_COUNT , False
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS , 0
[Step 9/11] Creating infer requests and preparing input data
[ INFO ] Create 1 infer requests took 0.13 ms
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests using 1 streams for CPU, inference only: True, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 5.28 ms
[Step 11/11] Dumping statistics report
Count: 51427 iterations
Duration: 60001.37 ms
Latency:
Median: 1.10 ms
AVG: 1.14 ms
MIN: 1.06 ms
MAX: 6.34 ms
Throughput: 857.10 FPS
# Inference INT8 model (IR)
!benchmark_app -m $compressed_model_xml -d CPU -api async
[Step 1/11] Parsing and validating input arguments
[ WARNING ] -nstreams default value is determined automatically for a device. Although the automatic selection usually provides a reasonable performance, but it still may be non-optimal for some cases, for more information look at README.
[Step 2/11] Loading OpenVINO
[ WARNING ] PerformanceMode was not explicitly specified in command line. Device CPU performance hint will be set to THROUGHPUT.
[ INFO ] OpenVINO:
API version............. 2022.1.0-7019-cdb9bec7210-releases/2022/1
[ INFO ] Device info
CPU
openvino_intel_cpu_plugin version 2022.1
Build................... 2022.1.0-7019-cdb9bec7210-releases/2022/1
[Step 3/11] Setting device configuration
[ WARNING ] -nstreams default value is determined automatically for CPU device. Although the automatic selection usually provides a reasonable performance, but it still may be non-optimal for some cases, for more information look at README.
[Step 4/11] Reading network files
[ INFO ] Read model took 26.05 ms
[Step 5/11] Resizing network to match image sizes and given batch
[ INFO ] Network batch size: 1
[Step 6/11] Configuring input of the model
[ INFO ] Model input 'input.1' precision u8, dimensions ([N,C,H,W]): 1 3 32 32
[ INFO ] Model output '466' precision f32, dimensions ([...]): 1 10
[Step 7/11] Loading the model to the device
[ INFO ] Compile model took 165.60 ms
[Step 8/11] Querying optimal runtime parameters
[ INFO ] DEVICE: CPU
[ INFO ] AVAILABLE_DEVICES , ['']
[ INFO ] RANGE_FOR_ASYNC_INFER_REQUESTS , (1, 1, 1)
[ INFO ] RANGE_FOR_STREAMS , (1, 2)
[ INFO ] FULL_DEVICE_NAME , Intel(R) Xeon(R) Platinum 8272CL CPU @ 2.60GHz
[ INFO ] OPTIMIZATION_CAPABILITIES , ['WINOGRAD', 'FP32', 'FP16', 'INT8', 'BIN', 'EXPORT_IMPORT']
[ INFO ] CACHE_DIR ,
[ INFO ] NUM_STREAMS , 1
[ INFO ] INFERENCE_NUM_THREADS , 0
[ INFO ] PERF_COUNT , False
[ INFO ] PERFORMANCE_HINT_NUM_REQUESTS , 0
[Step 9/11] Creating infer requests and preparing input data
[ INFO ] Create 1 infer requests took 0.15 ms
[ WARNING ] No input files were given for input 'input.1'!. This input will be filled with random values!
[ INFO ] Fill input 'input.1' with random values
[Step 10/11] Measuring performance (Start inference asynchronously, 1 inference requests using 1 streams for CPU, inference only: True, limits: 60000 ms duration)
[ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
[ INFO ] First inference took 11.03 ms
[Step 11/11] Dumping statistics report
Count: 59843 iterations
Duration: 60001.60 ms
Latency:
Median: 0.94 ms
AVG: 0.98 ms
MIN: 0.91 ms
MAX: 5.39 ms
Throughput: 997.36 FPS
Compare results on four pictures.¶
ie = Core()
# read and load float model
float_model = ie.read_model(
model=ir_model_xml, weights=ir_model_bin
)
float_compiled_model = ie.compile_model(model=float_model, device_name="CPU")
# read and load quantized model
quantized_model = ie.read_model(
model=compressed_model_xml, weights=compressed_model_bin
)
quantized_compiled_model = ie.compile_model(model=quantized_model, device_name="CPU")
# define all possible labels from CIFAR10
labels_names = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
all_pictures = []
all_labels = []
# get all pictures and their labels
for i, batch in enumerate(data_loader):
all_pictures.append(batch[1])
all_labels.append(batch[0][1])
def plot_pictures(indexes: list, all_pictures=all_pictures, all_labels=all_labels):
"""Plot 4 pictures.
:param indexes: a list of indexes of pictures to be displayed.
:param all_batches: batches with pictures.
"""
images, labels = [], []
num_pics = len(indexes)
assert num_pics == 4, f'No enough indexes for pictures to be displayed, got {num_pics}'
for idx in indexes:
assert idx < 10000, 'Cannot get such index, there are only 10000'
pic = np.rollaxis(all_pictures[idx].squeeze(), 0, 3)
images.append(pic)
labels.append(labels_names[all_labels[idx]])
f, axarr = plt.subplots(1, 4)
axarr[0].imshow(images[0])
axarr[0].set_title(labels[0])
axarr[1].imshow(images[1])
axarr[1].set_title(labels[1])
axarr[2].imshow(images[2])
axarr[2].set_title(labels[2])
axarr[3].imshow(images[3])
axarr[3].set_title(labels[3])
def infer_on_pictures(model, indexes: list, all_pictures=all_pictures):
""" Inference model on a few pictures.
:param net: model on which do inference
:param indexes: list of indexes
"""
output_key = model.output(0)
predicted_labels = []
for idx in indexes:
assert idx < 10000, 'Cannot get such index, there are only 10000'
result = model([all_pictures[idx][None,]])[output_key]
result = labels_names[np.argmax(result[0])]
predicted_labels.append(result)
return predicted_labels
indexes_to_infer = [7, 12, 15, 20] # to plot specify 4 indexes
plot_pictures(indexes_to_infer)
results_float = infer_on_pictures(float_compiled_model, indexes_to_infer)
results_quanized = infer_on_pictures(quantized_compiled_model, indexes_to_infer)
print(f"Labels for picture from float model : {results_float}.")
print(f"Labels for picture from quantized model : {results_quanized}.")
Labels for picture from float model : ['frog', 'dog', 'ship', 'horse'].
Labels for picture from quantized model : ['frog', 'dog', 'ship', 'horse'].
