Configuring Devices¶

Introduction¶

Inference Engine API provides the ability to configure devices via configuration keys and get device specific metrics. The values taken from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter, making users lost on what the actual type is stored in this parameter.

OpenVINO Runtime API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. Their main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

The property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets below explain how to migrate from an Inference Engine device configuration to OpenVINO Runtime API 2.0.

Set configuration values¶

Inference Engine API:

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");

auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

core.set_config({"PERF_COUNT": "YES"}, "CPU")

exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                "PERFORMANCE_HINT": "THROUGHPUT",
                                                "ENFORCE_BF16": "NO"})

# turn CPU off for multi-device execution
exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

OpenVINO Runtime API 2.0:

core.set_property("CPU", ov::enable_profiling(true));

auto compiled_model = core.compile_model(model, "MULTI",
    ov::device::priorities("GPU", "CPU"),
    ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
    ov::hint::inference_precision(ov::element::f32));

// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));

core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"})

compiled_model = core.compile_model(model=model, device_name="MULTI", config=
    {
        "MULTI_DEVICE_PRIORITIES": "GPU,CPU",
        "PERFORMANCE_HINT": "THROUGHPUT",
        "INFERENCE_PRECISION_HINT": "f32"
    })

# turn CPU off for multi-device execution
compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"})

Get Information¶

Inference Engine API:

// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();

auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();

std::string perf_model = exec_network.GetConfig(CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>();

auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();

num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")

full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")

perf_hint = exec_network.get_config("PERFORMANCE_HINT")

nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

OpenVINO Runtime API 2.0:

// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);

// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);

num_streams = core.get_property("CPU", "NUM_STREAMS")

full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME")

perf_mode = compiled_model.get_property("PERFORMANCE_HINT")

nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

Prev Next