1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
// Copyright (C) 2018 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
/**
* @brief a header file for IInferRequest interface
* @file ie_iinfer_request.hpp
*/
#pragma once
#include "ie_common.h"
#include <ie_blob.h>
#include <memory>
#include <string>
#include <map>
#include <details/ie_irelease.hpp>
namespace InferenceEngine {
/**
* @brief This is an interface of asynchronous infer request
*/
class IInferRequest : public details::IRelease {
public:
/**
* @enum WaitMode
* @brief Enumeration to hold wait mode for IInferRequest
*/
enum WaitMode : int64_t {
/** Wait until inference result becomes available */
RESULT_READY = -1,
/** IInferRequest doesn't block or interrupt current thread and immediately returns inference status */
STATUS_ONLY = 0,
};
using Ptr = std::shared_ptr<IInferRequest>;
using WeakPtr = std::weak_ptr<IInferRequest>;
/**
* @brief Sets input/output data to infer
* @note: Memory allocation does not happen
* @param name Name of input or output blob.
* @param data Reference to input or output blob. The type of a blob must match the network input precision and size.
* @param resp Optional: pointer to an already allocated object to contain information in case of failure
* @return Status code of the operation: OK (0) for success
*/
virtual StatusCode SetBlob(const char *name, const Blob::Ptr &data, ResponseDesc *resp) noexcept = 0;
/**
* @brief Gets input/output data for inference
* @note: Memory allocation does not happen
* @param name Name of input or output blob.
* @param data Reference to input or output blob. The type of Blob must match the network input precision and size.
* @param resp Optional: pointer to an already allocated object to contain information in case of failure
* @return Status code of the operation: OK (0) for success
*/
virtual StatusCode GetBlob(const char *name, Blob::Ptr &data, ResponseDesc *resp) noexcept = 0;
/**
* @brief Infers specified input(s) in synchronous mode
* @note blocks all methods of IInferRequest while request is ongoing (running or waiting in queue)
* @param resp Optional: pointer to an already allocated object to contain information in case of failure
* @return Status code of the operation: OK (0) for success
*/
virtual StatusCode Infer(ResponseDesc *resp) noexcept = 0;
/**
* @brief Queries performance measures per layer to get feedback of what is the most time consuming layer
* @note: not all plugins provide meaningful data
* @param perfMap Map of layer names to profiling information for that layer
* @param resp Optional: pointer to an already allocated object to contain information in case of failure
* @return Status code of the operation: OK (0) for success
*/
virtual StatusCode GetPerformanceCounts(std::map<std::string, InferenceEngineProfileInfo> &perfMap,
ResponseDesc *resp) const noexcept = 0;
/**
* @brief Waits for the result to become available. Blocks until specified millis_timeout has elapsed or the result becomes available, whichever comes first.
* @param millis_timeout Maximum duration in milliseconds to block for
* @note There are special cases when millis_timeout is equal some value of the WaitMode enum:
* * STATUS_ONLY - immediately returns inference status (IInferRequest::RequestStatus). It does not block or interrupt current thread
* * RESULT_READY - waits until inference result becomes available
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if occurred)
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual InferenceEngine::StatusCode Wait(int64_t millis_timeout, ResponseDesc *resp) noexcept = 0;
/**
* @brief Starts inference of specified input(s) in asynchronous mode
* @note: It returns immediately. Inference starts also immediately
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if occurred)
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual StatusCode StartAsync(ResponseDesc *resp) noexcept = 0;
/**
* @brief Completion callback definition as pointer to a function
* @param context Pointer to request for providing context inside callback
* @param code Completion result status: OK (0) for success
*/
typedef void (*CompletionCallback)(InferenceEngine::IInferRequest::Ptr context,
InferenceEngine::StatusCode code);
/**
* @brief Sets a callback function that will be called on success or failure of asynchronous request
* @param callback A function to be called
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual StatusCode SetCompletionCallback(CompletionCallback callback) noexcept = 0;
/**
* @brief Gets arbitrary data for the request and stores a pointer to a pointer to the obtained data
* @param data Pointer to a pointer to the gotten arbitrary data
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if occurred)
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual StatusCode GetUserData(void **data, ResponseDesc *resp) noexcept = 0;
/**
* @brief Sets arbitrary data for the request
* @param data Pointer to a pointer to arbitrary data to set
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if occurred)
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual StatusCode SetUserData(void *data, ResponseDesc *resp) noexcept = 0;
/**
* @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
* @param batch_size new batch size to be used by all the following inference calls for this request.
* @param resp Optional: a pointer to an already allocated object to contain extra information of a failure (if occurred)
* @return Enumeration of the resulted action: OK (0) for success
*/
virtual InferenceEngine::StatusCode SetBatch(int batch_size, ResponseDesc *resp) noexcept = 0;
};
} // namespace InferenceEngine
|
