diff options
Diffstat (limited to 'inference-engine/include')
28 files changed, 871 insertions, 669 deletions
diff --git a/inference-engine/include/cpp/ie_cnn_net_reader.h b/inference-engine/include/cpp/ie_cnn_net_reader.h index 6a15c810b..5a9b55a1d 100644 --- a/inference-engine/include/cpp/ie_cnn_net_reader.h +++ b/inference-engine/include/cpp/ie_cnn_net_reader.h @@ -75,7 +75,11 @@ public: CNNNetwork getNetwork() { // network obj are to be updated upon this call if (network.get() == nullptr) { - network.reset(new CNNNetwork(actual)); + try { + network.reset(new CNNNetwork(actual)); + } catch (...) { + THROW_IE_EXCEPTION << "Could not allocate memory"; + } } return *network.get(); } diff --git a/inference-engine/include/cpp/ie_cnn_network.h b/inference-engine/include/cpp/ie_cnn_network.h index 90420e414..680524731 100644 --- a/inference-engine/include/cpp/ie_cnn_network.h +++ b/inference-engine/include/cpp/ie_cnn_network.h @@ -191,7 +191,7 @@ public: * @brief - Helper method to get collect all input shapes with names of corresponding Data objects * @return Map of pairs: input's name and its dimension. */ - virtual ICNNNetwork::InputShapes getInputShapes() { + virtual ICNNNetwork::InputShapes getInputShapes() const { ICNNNetwork::InputShapes shapes; InputsDataMap inputs; actual->getInputsInfo(inputs); @@ -207,6 +207,10 @@ public: return std::move(shapes); } + /** + * @brief Run shape inference with new input shapes for the network + * @param inputShapes - map of pairs: name of corresponding data and its dimension. + */ virtual void reshape(const ICNNNetwork::InputShapes &inputShapes) { CALL_STATUS_FNC(reshape, inputShapes); } diff --git a/inference-engine/include/cpp/ie_memory_state.hpp b/inference-engine/include/cpp/ie_memory_state.hpp index 37799e8a6..99d32c867 100644 --- a/inference-engine/include/cpp/ie_memory_state.hpp +++ b/inference-engine/include/cpp/ie_memory_state.hpp @@ -51,4 +51,4 @@ class MemoryState { } }; -} // namespace InferenceEngine +} // namespace InferenceEngine
\ No newline at end of file diff --git a/inference-engine/include/details/caseless.hpp b/inference-engine/include/details/caseless.hpp new file mode 100644 index 000000000..b4a4ba225 --- /dev/null +++ b/inference-engine/include/details/caseless.hpp @@ -0,0 +1,81 @@ +// Copyright (C) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include <algorithm> +#include <functional> +#include <unordered_map> +#include <map> +#include <set> +#include <cctype> + +namespace InferenceEngine { +namespace details { + +/** + * @brief provides case-less comparison for stl algorithms + * @tparam Key type, usually std::string + */ +template<class Key> +class CaselessLess : public std::binary_function<Key, Key, bool> { + public: + bool operator () (const Key & a, const Key & b) const noexcept { + return std::lexicographical_compare(std::begin(a), + std::end(a), + std::begin(b), + std::end(b), + [](const char&cha, const char&chb) { + return std::tolower(cha) < std::tolower(chb); + }); + } +}; + +/** + * provides caseless eq for stl algorithms + * @tparam Key + */ +template<class Key> +class CaselessEq : public std::binary_function<Key, Key, bool> { + public: + bool operator () (const Key & a, const Key & b) const noexcept { + return a.size() == b.size() && + std::equal(std::begin(a), + std::end(a), + std::begin(b), + [](const char&cha, const char&chb) { + return std::tolower(cha) == std::tolower(chb); + }); + } +}; + +/** + * To hash caseless + */ +template<class T> +class CaselessHash : public std::hash<T> { + public: + size_t operator()(T __val) const noexcept { + T lc; + std::transform(std::begin(__val), std::end(__val), std::back_inserter(lc), [](typename T::value_type ch) { + return std::tolower(ch); + }); + return std::hash<T>()(lc); + } +}; + +template <class Key, class Value> +using caseless_unordered_map = std::unordered_map<Key, Value, CaselessHash<Key>, CaselessEq<Key>>; + +template <class Key, class Value> +using caseless_unordered_multimap = std::unordered_multimap<Key, Value, CaselessHash<Key>, CaselessEq<Key>>; + +template <class Key, class Value> +using caseless_map = std::map<Key, Value, CaselessLess<Key>>; + +template <class Key> +using caseless_set = std::set<Key, CaselessLess<Key>>; + +} // namespace details +} // namespace InferenceEngine diff --git a/inference-engine/include/details/ie_cnn_network_iterator.hpp b/inference-engine/include/details/ie_cnn_network_iterator.hpp index 733ff1339..a5c0e45bf 100644 --- a/inference-engine/include/details/ie_cnn_network_iterator.hpp +++ b/inference-engine/include/details/ie_cnn_network_iterator.hpp @@ -90,7 +90,7 @@ class CNNNetworkIterator { */ const CNNLayerPtr &operator*() const { if (nullptr == currentLayer) { - THROW_IE_EXCEPTION << "iterator of ouf bound"; + THROW_IE_EXCEPTION << "iterator out of bound"; } return currentLayer; } diff --git a/inference-engine/include/details/ie_cnn_network_tools.h b/inference-engine/include/details/ie_cnn_network_tools.h new file mode 100644 index 000000000..82a0c9246 --- /dev/null +++ b/inference-engine/include/details/ie_cnn_network_tools.h @@ -0,0 +1,21 @@ +// Copyright (C) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file for CNNNetwork tools + * @file ie_cnn_network_tools.h + */ +#pragma once +#include <vector> +#include "ie_common.h" +#include "ie_icnn_network.hpp" + +namespace InferenceEngine { +namespace details { + +INFERENCE_ENGINE_API_CPP(std::vector<CNNLayerPtr>) CNNNetSortTopologically(const ICNNNetwork & network); + +} // namespace details +} // namespace InferenceEngine diff --git a/inference-engine/include/details/ie_exception_conversion.hpp b/inference-engine/include/details/ie_exception_conversion.hpp index 582a62949..1ec3d041b 100644 --- a/inference-engine/include/details/ie_exception_conversion.hpp +++ b/inference-engine/include/details/ie_exception_conversion.hpp @@ -68,6 +68,7 @@ inline void extract_exception(StatusCode status, char *msg) { case RESULT_NOT_READY:throw ResultNotReady(msg); case NOT_ALLOCATED:throw NotAllocated(msg); case INFER_NOT_STARTED:throw InferNotStarted(msg); + case NETWORK_NOT_READ:throw NetworkNotRead(msg); default:THROW_IE_EXCEPTION << msg; } } diff --git a/inference-engine/include/details/ie_no_release.hpp b/inference-engine/include/details/ie_no_release.hpp index e2983981a..6e348ee82 100644 --- a/inference-engine/include/details/ie_no_release.hpp +++ b/inference-engine/include/details/ie_no_release.hpp @@ -22,4 +22,4 @@ class NoReleaseOn : public T { }; } // namespace details -} // namespace InferenceEngine +} // namespace InferenceEngine
\ No newline at end of file diff --git a/inference-engine/include/details/ie_pre_allocator.hpp b/inference-engine/include/details/ie_pre_allocator.hpp index 6c8819526..6953bc1d5 100644 --- a/inference-engine/include/details/ie_pre_allocator.hpp +++ b/inference-engine/include/details/ie_pre_allocator.hpp @@ -84,4 +84,4 @@ std::shared_ptr<IAllocator> make_pre_allocator(T *ptr, size_t size) { } } // namespace details -} // namespace InferenceEngine +} // namespace InferenceEngine
\ No newline at end of file diff --git a/inference-engine/include/hetero/hetero_plugin_config.hpp b/inference-engine/include/hetero/hetero_plugin_config.hpp index 0d9254a3d..249392edc 100644 --- a/inference-engine/include/hetero/hetero_plugin_config.hpp +++ b/inference-engine/include/hetero/hetero_plugin_config.hpp @@ -29,6 +29,7 @@ namespace HeteroConfigParams { * This option should be used with values: CONFIG_VALUE(NO) (default) or CONFIG_VALUE(YES) */ DECLARE_HETERO_CONFIG_KEY(DUMP_GRAPH_DOT); +DECLARE_HETERO_CONFIG_KEY(DUMP_DLA_MESSAGES); } // namespace HeteroConfigParams } // namespace InferenceEngine diff --git a/inference-engine/include/ie_blob.h b/inference-engine/include/ie_blob.h index 8a2d6b4f2..c6d02031a 100644 --- a/inference-engine/include/ie_blob.h +++ b/inference-engine/include/ie_blob.h @@ -120,7 +120,7 @@ public: : tensorDesc(p, SizeVector(dims.rbegin(), dims.rend()), l) {} /** - * @depricated It works with reversed dimensions. Please create a new blob if you want to change a size. + * @deprecated It works with reversed dimensions. Please create a new blob if you want to change a size. * @brief Changes Tensor size to the specified dimensions. If it was allocated, the previous data is deallocated and lost. * @param dims New dimensions to set * @param layout New layout to set @@ -290,11 +290,16 @@ public: * @param data_size Length of the pre-allocated array. If not set, size is assumed equal * to the dot product of dims. */ - TBlob(const TensorDesc& tensorDesc, T* ptr, size_t data_sze = 0): Blob(tensorDesc) { - if (size() != 0 && ptr == nullptr) { + TBlob(const TensorDesc& tensorDesc, T* ptr, size_t data_size = 0): Blob(tensorDesc) { + if (data_size == 0) { + data_size = size(); + } + + if (data_size != 0 && ptr == nullptr) { THROW_IE_EXCEPTION << "Using Blob on external nullptr memory"; } - _allocator = details::make_pre_allocator(ptr, size()); + + _allocator = details::make_pre_allocator(ptr, data_size); // blob on attached memory is always allocated, so we are not forcing the user to call allocate() allocate(); } @@ -327,11 +332,14 @@ public: * @param ptr Pointer to the pre-allocated memory * @param data_size Length of the pre-allocated array. If not set, size is assumed equal to dot product of dims. */ - TBlob(Precision p, Layout l, const SizeVector& dims, T* ptr, size_t data_sze = 0) : Blob(p, l, dims) { - if (size() != 0 && ptr == nullptr) { + TBlob(Precision p, Layout l, const SizeVector& dims, T* ptr, size_t data_size = 0) : Blob(p, l, dims) { + if (data_size == 0) { + data_size = size(); + } + if (data_size != 0 && ptr == nullptr) { THROW_IE_EXCEPTION << "Using Blob on external nullptr memory"; } - _allocator = details::make_pre_allocator(ptr, size()); + _allocator = details::make_pre_allocator(ptr, data_size); // blob on attached memory is always allocated, so we are not forcing user to call allocate allocate(); } @@ -416,7 +424,10 @@ public: if (tensorDesc.getDims().size() == 0) { tensorDesc.setDims({static_cast<unsigned int>(that.size())}); } - allocate(); + // minimisation of reallocations + if (_handle == nullptr) { + allocate(); + } auto memptr = data(); memcpy(memptr, that.data(), product(tensorDesc.getDims()) * sizeof(T)); } diff --git a/inference-engine/include/ie_common.h b/inference-engine/include/ie_common.h index 1ba426157..6489e8a7d 100644 --- a/inference-engine/include/ie_common.h +++ b/inference-engine/include/ie_common.h @@ -67,6 +67,12 @@ union UserValue { void *v_ptr; }; +enum CellType { + ORIG, + LSTM, + GRU +}; + /** * @enum Layout * @brief Layouts that the inference engine supports @@ -94,6 +100,29 @@ enum Layout : uint8_t { BLOCKED = 200, }; +inline std::ostream & operator << (std::ostream &out, const Layout & p) { + switch (p) { +#define PRINT_LAYOUT(name)\ + case name : out << #name; break; + + PRINT_LAYOUT(ANY); + PRINT_LAYOUT(NCHW); + PRINT_LAYOUT(NHWC); + PRINT_LAYOUT(OIHW); + PRINT_LAYOUT(C); + PRINT_LAYOUT(CHW); + PRINT_LAYOUT(HW); + PRINT_LAYOUT(NC); + PRINT_LAYOUT(CN); + PRINT_LAYOUT(BLOCKED); +#undef PRINT_LAYOUT + default: + out << static_cast<int>(p); + break; + } + return out; + } + /** * @struct InferenceEngineProfileInfo @@ -157,7 +186,8 @@ enum StatusCode : int { REQUEST_BUSY = -8, RESULT_NOT_READY = -9, NOT_ALLOCATED = -10, - INFER_NOT_STARTED = -11 + INFER_NOT_STARTED = -11, + NETWORK_NOT_READ = -12 }; /** @@ -216,6 +246,10 @@ class InferNotStarted : public std::logic_error { using std::logic_error::logic_error; }; } // namespace InferenceEngine +/** @brief This class represents StatusCode::NETWORK_NOT_READ exception */ +class NetworkNotRead : public std::logic_error +{ using std::logic_error::logic_error; }; + #if defined(_WIN32) #define __PRETTY_FUNCTION__ __FUNCSIG__ #else diff --git a/inference-engine/include/ie_data.h b/inference-engine/include/ie_data.h index 31ffb460a..818594e4c 100644 --- a/inference-engine/include/ie_data.h +++ b/inference-engine/include/ie_data.h @@ -104,7 +104,7 @@ public: * Batch is defined as the last element in the dimensions vector. * @param batch_size Batch size to set */ - inline void setBatchSize(size_t batch_size); + void setBatchSize(size_t batch_size); /** * @brief Sets the layout value for this Data instance diff --git a/inference-engine/include/ie_extension.h b/inference-engine/include/ie_extension.h index aed2ad234..871d8c99f 100644 --- a/inference-engine/include/ie_extension.h +++ b/inference-engine/include/ie_extension.h @@ -11,7 +11,6 @@ #include "details/ie_so_pointer.hpp" #include "ie_iextension.h" -#include "mkldnn/mkldnn_extension_ptr.hpp" #include <string> #include <memory> #include <map> @@ -166,8 +165,8 @@ public: * @param resp Response descriptor * @return Status code */ - StatusCode getPrimitiveTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept override { - return actual->getPrimitiveTypes(types, size, resp); + StatusCode getShapeInferTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept override { + return actual->getShapeInferTypes(types, size, resp); } /** @@ -204,11 +203,7 @@ inline std::shared_ptr<IShapeInferExtension> make_so_pointer(const std::string & */ template<> inline std::shared_ptr<IExtension> make_so_pointer(const std::string &name) { - try { - return std::make_shared<Extension>(name); - } catch (InferenceEngine::details::InferenceEngineException& ex) { - return std::make_shared<MKLDNNPlugin::MKLDNNExtension>(name); - } + return std::make_shared<Extension>(name); } } // namespace InferenceEngine diff --git a/inference-engine/include/ie_icnn_net_reader.h b/inference-engine/include/ie_icnn_net_reader.h index b444ad1ca..0a060b121 100644 --- a/inference-engine/include/ie_icnn_net_reader.h +++ b/inference-engine/include/ie_icnn_net_reader.h @@ -29,6 +29,7 @@ class ICNNNetReader : public details::IRelease { public: /** * @brief Parses the topology part of the IR (.xml) + * This method can be called once only to read network. If you need to read another network instance then create new reader instance. * @param filepath The full path to the .xml file of the IR * @param resp Response message * @return Result code @@ -37,6 +38,7 @@ public: /** * @brief Parses the topology part of the IR (.xml) given the xml as a buffer + * This method can be called once only to read network. If you need to read another network instance then create new reader instance. * @param model Pointer to a char array with the IR * @param resp Response message * @param size Size of the char array in bytes diff --git a/inference-engine/include/ie_icnn_network.hpp b/inference-engine/include/ie_icnn_network.hpp index 216131802..6aa1f84e3 100644 --- a/inference-engine/include/ie_icnn_network.hpp +++ b/inference-engine/include/ie_icnn_network.hpp @@ -17,6 +17,7 @@ #include "details/ie_irelease.hpp" #include "ie_preprocess.hpp" #include "ie_input_info.hpp" +#include "ie_icnn_network_stats.hpp" #include "ie_iextension.h" #include <memory> #include <map> @@ -28,6 +29,8 @@ namespace InferenceEngine { * @brief A collection that contains string as key, and Data smart pointer as value */ using OutputsDataMap = std::map<std::string, DataPtr>; +class IShapeInferExtension; +using IShapeInferExtensionPtr = std::shared_ptr<IShapeInferExtension>; /** * @brief This is the main interface to describe the NN topology @@ -143,9 +146,10 @@ public: * @note There are several limitations and it's not recommended to use it. Set batch to the input shape and call @reshape. * @param size Size of batch to set * @return Status code of the operation - * @note: Current implementation of the function sets batch size to the first dimension of 4D input layers in the networks - * and starts shape inference for IR starting from v3, for IR v2 it sets batch to the first dimension for all layers. - * Custom layers might require custom shape infer implementation, use @IShapeInferExtension interface to register them. + * @note: Current implementation of the function sets batch size to the first dimension of all layers in the networks. + * Before calling it make sure that all your layers have batch in the first dimension, otherwise the method works incorrectly. + * This limitation is resolved via [Shape Inference feature](./docs/Inference_Engine_Developer_Guide/ShapeInference.md) + * by using InferenceEngine::ICNNNetwork::reshape method. */ virtual StatusCode setBatchSize(size_t size, ResponseDesc* responseDesc) noexcept = 0; @@ -161,12 +165,11 @@ public: using InputShapes = std::map<std::string, SizeVector>; /** - * @brief - Run shape inference with new input shapes for the network - * @param inputShapes - map of pairs: name of corresponding data and its dimension. - * @note currently all inputs are required - * @param resp Pointer to the response message that holds a description of an error if any occurred - * @return Status code of the operation - */ + * @brief Run shape inference with new input shapes for the network + * @param inputShapes - map of pairs: name of corresponding data and its dimension. + * @param resp Pointer to the response message that holds a description of an error if any occurred + * @return Status code of the operation + */ virtual StatusCode reshape(const InputShapes& inputShapes, ResponseDesc* resp) noexcept { return NOT_IMPLEMENTED; }; /** @@ -177,5 +180,7 @@ public: */ virtual StatusCode AddExtension(const IShapeInferExtensionPtr& extension, ResponseDesc* resp) noexcept { return NOT_IMPLEMENTED; }; + + virtual StatusCode getStats(ICNNNetworkStats** stats, ResponseDesc* resp) const noexcept { return NOT_IMPLEMENTED; }; }; } // namespace InferenceEngine diff --git a/inference-engine/include/ie_icnn_network_stats.hpp b/inference-engine/include/ie_icnn_network_stats.hpp index 3748194fa..904a5f2b8 100644 --- a/inference-engine/include/ie_icnn_network_stats.hpp +++ b/inference-engine/include/ie_icnn_network_stats.hpp @@ -13,37 +13,39 @@ #include <memory> #include <limits> #include <vector> +#include <map> +#include "details/ie_irelease.hpp" namespace InferenceEngine { +class NetworkNodeStats; + +using NetworkNodeStatsPtr = std::shared_ptr<NetworkNodeStats>; +using NetworkNodeStatsWeakPtr = std::weak_ptr<NetworkNodeStats>; +using NetworkStatsMap = std::map<std::string, NetworkNodeStatsPtr>; /** * @class ICNNNetworkStats * @brief This is the interface to describe the NN topology scoring statistics */ class ICNNNetworkStats : public details::IRelease { public: - virtual void SaveToFile(const std::string& xmlPath, const std::string& binPath) const = 0; - virtual void LoadFromFile(const std::string& xmlPath, const std::string& binPath) = 0; + virtual void setNodesStats(const NetworkStatsMap& stats) = 0; + virtual const NetworkStatsMap& getNodesStats() const = 0; virtual bool isEmpty() const = 0; }; -class NetworkNodeStats; - -using NetworkNodeStatsPtr = std::shared_ptr<NetworkNodeStats>; -using NetworkNodeStatsWeakPtr = std::weak_ptr<NetworkNodeStats>; - class NetworkNodeStats { public: NetworkNodeStats() { } explicit NetworkNodeStats(int statCount) { - float min = std::numeric_limits<float>::max(); - float max = std::numeric_limits<float>::min(); + float mn = (std::numeric_limits<float>::max)(); + float mx = (std::numeric_limits<float>::min)(); for (int i = 0; i < statCount; i++) { - _minOutputs.push_back(min); - _maxOutputs.push_back(max); + _minOutputs.push_back(mn); + _maxOutputs.push_back(mx); } } diff --git a/inference-engine/include/ie_iextension.h b/inference-engine/include/ie_iextension.h index e0dd38713..54e0b7051 100644 --- a/inference-engine/include/ie_iextension.h +++ b/inference-engine/include/ie_iextension.h @@ -22,7 +22,6 @@ #include "details/ie_no_copy.hpp" - #if defined(_WIN32) && defined(IMPLEMENT_INFERENCE_EXTENSION_API) #define INFERENCE_EXTENSION_API(TYPE) extern "C" __declspec(dllexport) TYPE #else @@ -137,7 +136,9 @@ public: * @return Status code */ virtual StatusCode getShapes(const std::vector<TensorDesc>& inShapes, std::vector<TensorDesc>& outShapes, - ResponseDesc* resp) noexcept = 0; + ResponseDesc* resp) noexcept { + return NOT_IMPLEMENTED; + } /** * @brief Gets all possible implementations for the given cnn Layer @@ -156,6 +157,8 @@ class IShapeInferImpl { public: using Ptr = std::shared_ptr<IShapeInferImpl>; + virtual ~IShapeInferImpl() = default; + /** * @brief check that reshape can be applied, that parameters and shapes are valid */ @@ -191,13 +194,13 @@ public: virtual void Unload() noexcept = 0; /** - * @brief Gets the array with types of layers which are included in the extension + * @brief Fills passed array with types of layers which shape infer implementations are included in the extension * @param types Array to store the layer types * @param size Size of the layer types array * @param resp Response descriptor * @return Status code */ - virtual StatusCode getPrimitiveTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept = 0; + virtual StatusCode getShapeInferTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept = 0; /** * @brief Gets shape propagation implementation for the given string-type of cnn Layer @@ -218,9 +221,20 @@ public: virtual StatusCode getFactoryFor(ILayerImplFactory*& factory, const CNNLayer* cnnLayer, ResponseDesc* resp) noexcept = 0; - StatusCode getShapeInferImpl(IShapeInferImpl::Ptr& impl, - const char* type, - ResponseDesc* resp) noexcept override { + /** + * @brief Fills passed array with types of layers which kernel implementations are included in the extension + * @param types Array to store the layer types + * @param size Size of the layer types array + * @param resp Response descriptor + * @return Status code + */ + virtual StatusCode getPrimitiveTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept = 0; + + StatusCode getShapeInferTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept override { + return NOT_IMPLEMENTED; + }; + + StatusCode getShapeInferImpl(IShapeInferImpl::Ptr& impl, const char* type, ResponseDesc* resp) noexcept override { return NOT_IMPLEMENTED; }; }; diff --git a/inference-engine/include/ie_ihetero_plugin.hpp b/inference-engine/include/ie_ihetero_plugin.hpp index df7fd8c55..a202fb39a 100644 --- a/inference-engine/include/ie_ihetero_plugin.hpp +++ b/inference-engine/include/ie_ihetero_plugin.hpp @@ -84,6 +84,8 @@ public: QueryNetworkResult &res) noexcept { QueryNetwork(device, network, res); }; + + virtual void SetLogCallback(IErrorListener &listener) = 0; }; using MapDeviceLoaders = std::map<std::string, InferenceEngine::IHeteroDeviceLoader::Ptr>; diff --git a/inference-engine/include/ie_imemory_state.hpp b/inference-engine/include/ie_imemory_state.hpp index 02b489984..f2bfbf811 100644 --- a/inference-engine/include/ie_imemory_state.hpp +++ b/inference-engine/include/ie_imemory_state.hpp @@ -58,4 +58,4 @@ class IMemoryState : public details::no_copy { virtual StatusCode GetLastState(Blob::CPtr & lastState, ResponseDesc *resp) const noexcept = 0; }; -} // namespace InferenceEngine +} // namespace InferenceEngine
\ No newline at end of file diff --git a/inference-engine/include/ie_layers.h b/inference-engine/include/ie_layers.h index 6ee6e61ec..46d5d3efc 100644 --- a/inference-engine/include/ie_layers.h +++ b/inference-engine/include/ie_layers.h @@ -20,6 +20,9 @@ #include "ie_data.h" #include "ie_blob.h" #include "ie_device.hpp" +#include "ie_layers_property.hpp" + +#include "ie_icnn_network.hpp" namespace InferenceEngine { /** @@ -460,107 +463,91 @@ public: }; /** + * @brief convinenent way to declare property with backward compatibility to 2D members + */ +#define DEFINE_PROP(prop_name) \ +PropertyVector<unsigned int> prop_name;\ +unsigned int &prop_name##_x = prop_name.at(X_AXIS);\ +unsigned int &prop_name##_y = prop_name.at(Y_AXIS);\ + +/** * @brief This class represents a standard 3D Convolution Layer */ class ConvolutionLayer : public WeightableLayer { public: /** - * @brief A convolution kernel width + * @brief A convolution kernel array [X, Y, Z, ...] */ - unsigned int _kernel_x = 0; + DEFINE_PROP(_kernel); /** - * @brief A convolution kernel height + * @brief A convolution paddings begin array [X, Y, Z, ...] */ - unsigned int _kernel_y = 0; + DEFINE_PROP(_padding); /** - * @brief An input convolution stride width + * @brief A convolution paddings end array [X, Y, Z, ...] */ - unsigned int _stride_x = 1; + PropertyVector<unsigned int> _pads_end; /** - * @brief An Input convolution stride height + * @brief A convolution strides array [X, Y, Z, ...] */ - unsigned int _stride_y = 1; + DEFINE_PROP(_stride); /** - * @brief A number of output feature maps (size) generating the 3'rd output dimension + * @brief A convolution dilations array [X, Y, Z, ...] */ - unsigned int _out_depth = 0; + DEFINE_PROP(_dilation); /** - * @brief Input padding width + * @brief A number of output feature maps (size) generating the 3'rd output dimension */ - unsigned int _padding_x = 0; + unsigned int _out_depth = 0u; /** - * @brief Input padding height + * @brief Number of groups */ - unsigned int _padding_y = 0; + unsigned int _group = 1u; + /** - * @brief Dilation width + * @brief Creates a new ConvolutionLayer instance. */ - unsigned int _dilation_x = 1; + explicit ConvolutionLayer(const LayerParams &p) : WeightableLayer(p), + _kernel(2, 0u), _padding(2, 0u), _stride(2, 1u), _dilation(2, 1u) {} + /** + * @brief assignment operator + */ + ConvolutionLayer & operator = (const ConvolutionLayer & that) { + if (&that != this) { + WeightableLayer::operator=(that); + _kernel = that._kernel; + _padding = that._padding; + _pads_end = that._pads_end; + _stride = that._stride; + _dilation = that._dilation; + _out_depth = that._out_depth; + _group = that._group; + } + return *this; + } /** - * @brief Dilation height + * @brief move assignment operator */ - unsigned int _dilation_y = 1; + ConvolutionLayer& operator = (ConvolutionLayer &&) = default; /** - * @brief Number of groups + * @brief copy constructor */ - unsigned int _group = 1; - + ConvolutionLayer(const ConvolutionLayer & that) : WeightableLayer(that) { + operator = (that); + } /** - * @brief Creates a new ConvolutionLayer instance. + * @brief move constructor */ - using WeightableLayer::WeightableLayer; + ConvolutionLayer(ConvolutionLayer &&) = default; }; /** * @brief This class represents a standard deconvolution layer */ -class DeconvolutionLayer : public WeightableLayer { -public: - /** - * @brief Deconvolution kernel width - */ - unsigned int _kernel_x = 0; - /** - * @brief Deconvolution kernel height - */ - unsigned int _kernel_y = 0; - /** - * @brief Input Deconvolution stride width - */ - unsigned int _stride_x = 0; - /** - * @brief Input Deconvolution stride height - */ - unsigned int _stride_y = 0; - /** - * @brief number of output feature maps (size) generating the 3'rd output dimension - */ - unsigned int _out_depth = 0; - /** - * @brief Input padding width - */ - unsigned int _padding_x = 0; - /** - * @brief Input padding height - */ - unsigned int _padding_y = 0; - /** - * @brief Dilation width - */ - unsigned int _dilation_x = 0; - /** - * @brief Dilation height - */ - unsigned int _dilation_y = 0; - /** - * @brief Number of groups - */ - unsigned int _group = 0; - - /** - * @brief Creates a new DeconvolutionLayer instance. - */ - using WeightableLayer::WeightableLayer; +class DeconvolutionLayer : public ConvolutionLayer { + public: + using ConvolutionLayer::ConvolutionLayer; + using ConvolutionLayer::operator=; }; /** @@ -569,29 +556,21 @@ public: class PoolingLayer : public CNNLayer { public: /** - * @brief Pooling kernel width + * @brief Pooling kernel array [X, Y, Z, ...] */ - unsigned int _kernel_x = 0; + DEFINE_PROP(_kernel); /** - * @brief Pooling kernel height + * @brief Pooling paddings begin array [X, Y, Z, ...] */ - unsigned int _kernel_y = 0; + DEFINE_PROP(_padding); /** - * @brief Input Pooling stride width + * @brief Pooling paddings end array [X, Y, Z, ...] */ - unsigned int _stride_x = 0; + PropertyVector<unsigned int> _pads_end; /** - * @brief Input Pooling stride height + * @brief Pooling strides array [X, Y, Z, ...] */ - unsigned int _stride_y = 0; - /** - * @brief Input padding width - */ - unsigned int _padding_x = 0; - /** - * @brief Input padding height - */ - unsigned int _padding_y = 0; + DEFINE_PROP(_stride); /** * @enum PoolType @@ -618,9 +597,44 @@ public: /** * @brief Creates a new PoolingLayer instance. */ - using CNNLayer::CNNLayer; + explicit PoolingLayer(const LayerParams &p) : CNNLayer(p), + _kernel(2, 0u), _padding(2, 0u), _stride(2, 0u) {} + + /** + * @brief assignment operator + */ + PoolingLayer & operator = (const PoolingLayer & that) { + if (&that != this) { + CNNLayer::operator=(that); + _kernel = that._kernel; + _padding = that._padding; + _pads_end = that._pads_end; + _stride = that._stride; + _type = that._type; + _exclude_pad = that._exclude_pad; + } + return *this; + } + /** + * @brief move assignment operator + */ + PoolingLayer& operator = (PoolingLayer &&) = default; + + /** + * @brief copy constructor + */ + PoolingLayer(const PoolingLayer & that) : CNNLayer(that) { + operator=(that); + } + + /** + * @brief move constructor + */ + PoolingLayer(PoolingLayer &&) = default; }; +#undef DEFINE_PROP + /** * @brief This class represents a fully connected layer */ @@ -836,7 +850,6 @@ public: */ std::vector<int> axis; /** - * @deprecated result size is defined by second input * @brief A vector of dimensions to be preserved */ std::vector<int> dim; @@ -913,6 +926,66 @@ public: }; /** +* @brief This class represents RNN sequence layer +*/ +class RNNLayer : public WeightableLayer { +public: + CellType cellType; + + /** + * @brief An axis by which iteration is performed. Axis=0 means first input blob dimension is sequence, axis=1 means first dimension is batch. + */ + unsigned int _axis = 1; + + using WeightableLayer::WeightableLayer; + + /** + * @brief Creates a new RNNLayer instance. + */ + explicit RNNLayer(const LayerParams &p) : WeightableLayer(p) {} +}; + +/** +* @brief This class represents LSTMCell pseudo-layer to be used in TensorIterator +*/ +class LSTMCell : public WeightableLayer { +public: + using WeightableLayer::WeightableLayer; +}; + +class ICNNNetReader; +/** +* @brief This class represents TensorIterator layer +*/ +class TensorIterator : public CNNLayer { +public: + using CNNNetReaderPtr = std::shared_ptr<ICNNNetReader>; + CNNNetReaderPtr reader; + + struct BackEdge { + int fromLayer; + int fromPort; + int toLayer; + int toPort; + }; + + struct Port { + int external_port_id; + int internal_layer_id; + int internal_port_id; + int axis; + int part_size; + int stride; + }; + + std::vector<Port> input_ports; + std::vector<Port> output_ports; + std::vector<BackEdge> backEdges; + + using CNNLayer::CNNLayer; +}; + +/** * @class PReLULayer * @brief This class represents a Layer which performs Scale and Shift */ diff --git a/inference-engine/include/ie_layers_property.hpp b/inference-engine/include/ie_layers_property.hpp new file mode 100644 index 000000000..d0dcc8a57 --- /dev/null +++ b/inference-engine/include/ie_layers_property.hpp @@ -0,0 +1,125 @@ +// Copyright (C) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file for describing property style structure used by CNNLayers + * @file ie_layers_property.h + */ +#pragma once + +namespace InferenceEngine { + +constexpr const int MAX_DIMS_NUMBER = 12; + +enum eDIMS_AXIS : uint8_t { + X_AXIS = 0, + Y_AXIS, + Z_AXIS +}; + +template<class T, int N = MAX_DIMS_NUMBER> +class PropertyVector { + T _axises[N] = {}; + bool _allocated[N] = {}; + size_t _length = 0; + +public: + PropertyVector() = default; + + PropertyVector(size_t len, T val) { + if (len > N) { + THROW_IE_EXCEPTION << "Property size exceeed limit of: " << N; + } + for (int i = 0; i < len; i++) { + _axises[i] = val; + _allocated[i] = true; + } + _length = len; + } + + /** + * @brief allows access up-to capacity size + * @param index + * @return + */ + T &at(int index) { + if (index >= N) { + THROW_IE_EXCEPTION << "Property index is out of bounds(" << index << "/" << N; + } + return _axises[index]; + } + + const T &operator[](size_t index) const { + if (index >= N ||!_allocated[index]) { + THROW_IE_EXCEPTION << "Property index ("<< index <<")is out of bounds"; + } + return _axises[index]; + } + + T &operator[](size_t index) { + if (index >= N || !_allocated[index]) { + THROW_IE_EXCEPTION << "Property index ("<< index <<")is out of bounds"; + } + return _axises[index]; + } + + PropertyVector &operator=(const PropertyVector &src) { + if (this != &src) { + _length = src.size(); + for (size_t i = 0; i < N; i++) { + _allocated[i] = src._allocated[i]; + if (_allocated[i]) { + _axises[i] = src[i]; + } + } + } + return *this; + } + + bool operator==(const PropertyVector& src) const { + if (this == &src) return true; + if (_length != src.size()) return false; + for (size_t i = 0; i < N; i++) + if ((_allocated[i] != src._allocated[i]) || + (_allocated[i] && _axises[i] != src._axises[i])) return false; + return true; + } + + size_t size() const { + return _length; + } + + void insert(size_t axis, const T &val) { + if (axis < N) { + if (!_allocated[axis]) { + _allocated[axis] = true; + _length++; + } + _axises[axis] = val; + } else { + THROW_IE_EXCEPTION << "Layer Property insertion at(axis) should be in [0,"<< N<< ")"; + } + } + + void remove(size_t axis) { + if (axis < N && _allocated[axis]) { + _allocated[axis] = false; + _length--; + } + } + + void clear() { + for (int i = 0; i != N; i++) { + _allocated[i] = 0; + } + _length = 0u; + } + + bool exist(size_t axis) const { + return (axis < N && _allocated[axis]); + } +}; + +} // namespace InferenceEngine diff --git a/inference-engine/include/ie_parallel.hpp b/inference-engine/include/ie_parallel.hpp new file mode 100644 index 000000000..708da98f2 --- /dev/null +++ b/inference-engine/include/ie_parallel.hpp @@ -0,0 +1,354 @@ +// Copyright (C) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief Contains declarations and definitions for sequential and multi-threading implementations. + * Multi-threading support is implemented in two variants: using the Threading Building Blocks library and OpenMP* product. + * To build a particular implementation, use the corresponding identifier: IE_THREAD_TBB, IE_THREAD_OMP or IE_THREAD_SEQ. + * @file ie_parallel.hpp + */ + +#pragma once + +#define IE_THREAD_TBB 0 +#define IE_THREAD_OMP 1 +#define IE_THREAD_SEQ 2 + +#if IE_THREAD == IE_THREAD_TBB +#include "tbb/parallel_for.h" +#include "tbb/task_arena.h" + +#include "tbb/parallel_reduce.h" +#include "tbb/blocked_range.h" +#include "tbb/blocked_range2d.h" + +inline int parallel_get_max_threads() { return tbb::this_task_arena::max_concurrency(); } +inline int parallel_get_num_threads() { return parallel_get_max_threads(); } +inline int parallel_get_thread_num() { return tbb::this_task_arena::current_thread_index(); } +inline void parallel_set_num_threads(int n) { return; } + +#elif IE_THREAD == IE_THREAD_OMP +#include <omp.h> +/* MSVC still supports omp 2.0 only */ +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +# define collapse(x) +#endif // defined(_MSC_VER) && !defined(__INTEL_COMPILER) +inline int parallel_get_max_threads() { return omp_get_max_threads(); } +inline int parallel_get_num_threads() { return omp_get_num_threads(); } +inline int parallel_get_thread_num() { return omp_get_thread_num(); } +inline void parallel_set_num_threads(int n) { omp_set_num_threads(n); } + +#elif IE_THREAD == IE_THREAD_SEQ +inline int parallel_get_max_threads() { return 1; } +inline int parallel_get_num_threads() { return 1; } +inline int parallel_get_thread_num() { return 0; } +inline void parallel_set_num_threads(int n) { return; } +#endif + + +namespace InferenceEngine { + +template <typename F> +void parallel_nt(int nthr, F func) { +#if IE_THREAD == IE_THREAD_TBB + if (nthr == 0) nthr = parallel_get_max_threads(); + if (nthr == 1) { + func(0, 1); + return; + } + + tbb::parallel_for(0, nthr, [&](int ithr) { + func(ithr, nthr); + }); +#elif IE_THREAD == IE_THREAD_OMP + if (nthr == 1) { + func(0, 1); + return; + } + +# pragma omp parallel num_threads(nthr) + func(parallel_get_thread_num(), parallel_get_num_threads()); +#elif IE_THREAD == IE_THREAD_SEQ + func(0, 1); +#endif +} + +template <typename T0, typename R, typename F> +R parallel_sum(const T0 D0, R &input, F func) { +#if IE_THREAD == IE_THREAD_TBB + return tbb::parallel_reduce( + tbb::blocked_range<T0>(0, D0), input, + [&](const tbb::blocked_range<T0>& r, R init)->R { + R sum = init; + for (T0 dim1 = r.begin(); dim1 < r.end(); ++dim1) + sum += func(dim1); + return sum; + }, + [](R x, R y)->R { + return x + y; + }); +#else + R sum = input; +#if IE_THREAD == IE_THREAD_OMP + #pragma omp parallel for reduction(+ : sum) schedule(static) +#endif + for (T0 dim1 = 0; dim1 < D0; dim1++) { + sum += func(dim1); + } + return sum; +#endif +} + +template <typename T0, typename T1, typename R, typename F> +R parallel_sum2d(const T0 D0, const T1 D1, R input, F func) { +#if IE_THREAD == IE_THREAD_TBB + return tbb::parallel_reduce( + tbb::blocked_range2d<T0, T1>(0, D0, 0, D1), input, + [&](const tbb::blocked_range2d<T0, T1>& r, R init)->R { + R sum = init; + for (T0 dim2 = r.rows().begin(); dim2 < r.rows().end(); dim2++) { + for (T1 dim1 = r.cols().begin(); dim1 < r.cols().end(); dim1++) { + sum += func(dim2, dim1); + } + } + return sum; + }, + [](R x, R y)->R { + return x + y; + }); +#else + R sum = input; +#if IE_THREAD == IE_THREAD_OMP + #pragma omp parallel for collapse(2) reduction(+ : sum) schedule(static) +#endif + for (T0 dim2 = 0; dim2 < D0; dim2++) { + for (T1 dim1 = 0; dim1 < D1; dim1++) { + sum += func(dim2, dim1); + } + } + return sum; +#endif +} + +template<typename T> +inline T parallel_it_init(T start) { return start; } +template<typename T, typename Q, typename R, typename... Args> +inline T parallel_it_init(T start, Q &x, const R &X, Args &&... tuple) { + start = parallel_it_init(start, static_cast<Args>(tuple)...); + x = start % X; + return start / X; +} + +inline bool parallel_it_step() { return true; } +template<typename Q, typename R, typename... Args> +inline bool parallel_it_step(Q &x, const R &X, Args &&... tuple) { + if (parallel_it_step(static_cast<Args>(tuple)...)) { + x = (x + 1) % X; + return x == 0; + } + return false; +} + +template <typename T, typename Q> +inline void splitter(T n, Q team, Q tid, T &n_start, T &n_end) { + if (team <= 1 || n == 0) { + n_start = 0; + n_end = n; + } else { + T n1 = (n + (T)team - 1) / (T)team; + T n2 = n1 - 1; + T T1 = n - n2 * (T)team; + n_end = (T)tid < T1 ? n1 : n2; + n_start = (T)tid <= T1 ? tid * n1 : T1 * n1 + ((T)tid - T1) * n2; + } + + n_end += n_start; +} + + +template <typename T0, typename F> +void for_1d(const int ithr, const int nthr, const T0 &D0, F func) { + T0 d0{ 0 }, end{ 0 }; + splitter(D0, nthr, ithr, d0, end); + for (; d0 < end; ++d0) func(d0); +} + +template <typename T0, typename F> +void parallel_for(const T0 &D0, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_1d(ithr, nthr, D0, func); + }); +#elif IE_THREAD == IE_THREAD_OMP + # pragma omp parallel + for_1d(parallel_get_thread_num(), parallel_get_num_threads(), D0, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_1d(0, 1, D0, func); +#endif +} + + +template <typename T0, typename T1, typename F> +void for_2d(const int ithr, const int nthr, const T0 &D0, const T1 &D1, F func) { + const size_t work_amount = (size_t)D0 * D1; + if (work_amount == 0) return; + size_t start{ 0 }, end{ 0 }; + splitter(work_amount, nthr, ithr, start, end); + + T0 d0{ 0 }; T1 d1{ 0 }; + parallel_it_init(start, d0, D0, d1, D1); + for (size_t iwork = start; iwork < end; ++iwork) { + func(d0, d1); + parallel_it_step(d0, D0, d1, D1); + } +} + +template <typename T0, typename T1, typename F> +void parallel_for2d(const T0 &D0, const T1 &D1, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_2d(ithr, nthr, D0, D1, func); + }); +#elif IE_THREAD == IE_THREAD_OMP + # pragma omp parallel + for_2d(parallel_get_thread_num(), parallel_get_num_threads(), D0, D1, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_2d(0, 1, D0, D1, func); +#endif +} + + +template <typename T0, typename T1, typename T2, typename F> +void for_3d(const int ithr, const int nthr, const T0 &D0, const T1 &D1, + const T2 &D2, F func) { + const size_t work_amount = (size_t)D0 * D1 * D2; + if (work_amount == 0) return; + size_t start{ 0 }, end{ 0 }; + splitter(work_amount, nthr, ithr, start, end); + + T0 d0{ 0 }; T1 d1{ 0 }; T2 d2{ 0 }; + parallel_it_init(start, d0, D0, d1, D1, d2, D2); + for (size_t iwork = start; iwork < end; ++iwork) { + func(d0, d1, d2); + parallel_it_step(d0, D0, d1, D1, d2, D2); + } +} + +template <typename T0, typename T1, typename T2, typename F> +void parallel_for3d(const T0 &D0, const T1 &D1, const T2 &D2, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_3d(ithr, nthr, D0, D1, D2, func); + }); +#elif IE_THREAD == IE_THREAD_OMP + # pragma omp parallel + for_3d(parallel_get_thread_num(), parallel_get_num_threads(), D0, D1, D2, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_3d(0, 1, D0, D1, D2, func); +#endif +} + +template <typename T0, typename T1, typename T2, typename T3, typename F> +void for_4d(const int ithr, const int nthr, const T0 &D0, const T1 &D1, + const T2 &D2, const T3 &D3, F func) { + const size_t work_amount = (size_t)D0 * D1 * D2 * D3; + if (work_amount == 0) return; + size_t start{ 0 }, end{ 0 }; + splitter(work_amount, nthr, ithr, start, end); + + T0 d0{ 0 }; T1 d1{ 0 }; T2 d2{ 0 }; T3 d3{ 0 }; + parallel_it_init(start, d0, D0, d1, D1, d2, D2, d3, D3); + for (size_t iwork = start; iwork < end; ++iwork) { + func(d0, d1, d2, d3); + parallel_it_step(d0, D0, d1, D1, d2, D2, d3, D3); + } +} + +template <typename T0, typename T1, typename T2, typename T3, typename F> +void parallel_for4d(const T0 &D0, const T1 &D1, const T2 &D2, const T3 &D3, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_4d(ithr, nthr, D0, D1, D2, D3, func); + }); +#elif IE_THREAD == IE_THREAD_OMP + # pragma omp parallel + for_4d(parallel_get_thread_num(), parallel_get_num_threads(), D0, D1, D2, D3, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_4d(0, 1, D0, D1, D2, D3, func); +#endif +} + +template <typename T0, typename T1, typename T2, typename T3, typename T4, typename F> +void for_5d(const int ithr, const int nthr, const T0 &D0, const T1 &D1, + const T2 &D2, const T3 &D3, const T4 &D4, F func) { + const size_t work_amount = (size_t)D0 * D1 * D2 * D3 * D4; + if (work_amount == 0) return; + size_t start{ 0 }, end{ 0 }; + splitter(work_amount, nthr, ithr, start, end); + + T0 d0{ 0 }; T1 d1{ 0 }; T2 d2{ 0 }; T3 d3{ 0 }; T4 d4{ 0 }; + parallel_it_init(start, d0, D0, d1, D1, d2, D2, d3, D3, d4, D4); + for (size_t iwork = start; iwork < end; ++iwork) { + func(d0, d1, d2, d3, d4); + parallel_it_step(d0, D0, d1, D1, d2, D2, d3, D3, d4, D4); + } +} + +template <typename T0, typename T1, typename T2, typename T3, typename T4, typename F> +void parallel_for5d(const T0 &D0, const T1 &D1, const T2 &D2, const T3 &D3, + const T4 &D4, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_5d(ithr, nthr, D0, D1, D2, D3, D4, func); + }); +#elif IE_THREAD == IE_THREAD_OMP + # pragma omp parallel + for_5d(parallel_get_thread_num(), parallel_get_num_threads(), D0, D1, D2, D3, D4, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_5d(0, 1, D0, D1, D2, D3, D4, func); +#endif +} + + +template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename F> +void for_6d(const int ithr, const int nthr, const T0 &D0, const T1 &D1, + const T2 &D2, const T3 &D3, const T4 &D4, const T5 &D5, F func) { + const size_t work_amount = (size_t)D0 * D1 * D2 * D3 * D4 * D5; + if (work_amount == 0) return; + size_t start{ 0 }, end{ 0 }; + splitter(work_amount, nthr, ithr, start, end); + + T0 d0{ 0 }; T1 d1{ 0 }; T2 d2{ 0 }; T3 d3{ 0 }; T4 d4{ 0 }; T5 d5{ 0 }; + parallel_it_init(start, d0, D0, d1, D1, d2, D2, d3, D3, d4, D4, + d5, D5); + for (size_t iwork = start; iwork < end; ++iwork) { + func(d0, d1, d2, d3, d4, d5); + parallel_it_step(d0, D0, d1, D1, d2, D2, d3, D3, d4, D4, d5, D5); + } +} + +template <typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename F> +void parallel_for6d(const T0 &D0, const T1 &D1, const T2 &D2, const T3 &D3, + const T4 &D4, const T5 &D5, F func) { +#if IE_THREAD == IE_THREAD_TBB + const int nthr = parallel_get_max_threads(); + tbb::parallel_for(0, nthr, [&](int ithr) { + for_6d(ithr, nthr, D0, D1, D2, D3, D4, D5, func); + }); +#elif IE_THREAD == IE_THREAD_OMP +# pragma omp parallel + for_6d(parallel_get_thread_num(), parallel_get_num_threads(), D0, D1, D2, D3, D4, D5, func); +#elif IE_THREAD == IE_THREAD_SEQ + for_6d(0, 1, D0, D1, D2, D3, D4, D5, func); +#endif +} + +} // namespace InferenceEngine + diff --git a/inference-engine/include/inference_engine.hpp b/inference-engine/include/inference_engine.hpp index 7abfd11d4..08f043e37 100644 --- a/inference-engine/include/inference_engine.hpp +++ b/inference-engine/include/inference_engine.hpp @@ -21,6 +21,8 @@ #include <ie_device.hpp> #include <ie_plugin_dispatcher.hpp> #include <ie_plugin_config.hpp> +#include <ie_icnn_network.hpp> +#include <ie_icnn_network_stats.hpp> #include <cpp/ie_cnn_net_reader.h> #include <cpp/ie_plugin_cpp.hpp> #include <cpp/ie_executable_network.hpp> @@ -177,5 +179,4 @@ void copyToFloat(float *dst, const InferenceEngine::Blob *src) { for (size_t i = 0; i < t_blob->size(); i++) dst[i] = srcPtr[i]; } - } // namespace InferenceEngine diff --git a/inference-engine/include/mkldnn/mkldnn_extension.hpp b/inference-engine/include/mkldnn/mkldnn_extension.hpp deleted file mode 100644 index e14506d81..000000000 --- a/inference-engine/include/mkldnn/mkldnn_extension.hpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (C) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief A header file for the main MKL-DNN Extension API - * @file mkldnn_extension.hpp - */ -#pragma once - -#include <ie_iextension.h> - -#include "mkldnn_generic_primitive.hpp" - -namespace InferenceEngine { -namespace MKLDNNPlugin { - -/** - * @deprecated use new extensibility API - * @brief The IMKLDNNExtension class provides the main extension interface - */ -class IMKLDNNExtension : public IExtension { -public: - /** - * @brief Creates a generic layer and returns a pointer to an instance - * @param primitive Pointer to newly created layer - * @param layer Layer parameters (source for name, type, precision, attr, weights...) - * @param resp Optional: pointer to an already allocated object to contain information in case of failure - * @return Status code of the operation: OK (0) for success - */ - virtual InferenceEngine::StatusCode CreateGenericPrimitive(IMKLDNNGenericPrimitive*& primitive, - const InferenceEngine::CNNLayerPtr& layer, - InferenceEngine::ResponseDesc *resp) const noexcept = 0; - /** - * @brief This method isn't implemented for the old API - */ - StatusCode getPrimitiveTypes(char**& types, unsigned int& size, ResponseDesc* resp) noexcept override { - return NOT_IMPLEMENTED; - }; - /** - * @brief This method isn't implemented for the old API - */ - StatusCode getFactoryFor(ILayerImplFactory *&factory, const CNNLayer *cnnLayer, ResponseDesc *resp) noexcept override { - return NOT_IMPLEMENTED; - } - - /** - * @brief Gets shape propagation implementation for the given string-type of cnn Layer - * @param impl the vector with implementations which is ordered by priority - * @param resp response descriptor - * @return status code - */ - StatusCode getShapeInferImpl(IShapeInferImpl::Ptr& impl, const char* type, ResponseDesc* resp) noexcept override { - return NOT_IMPLEMENTED; - }; -}; - -/** - * @deprecated use new extensibility API - * @brief Creates the default instance of the extension - * @return The MKL-DNN Extension interface - */ -INFERENCE_EXTENSION_API(StatusCode) CreateMKLDNNExtension(IMKLDNNExtension*& ext, ResponseDesc* resp) noexcept; - -} // namespace MKLDNNPlugin -} // namespace InferenceEngine diff --git a/inference-engine/include/mkldnn/mkldnn_extension_ptr.hpp b/inference-engine/include/mkldnn/mkldnn_extension_ptr.hpp deleted file mode 100644 index 70ab1f66f..000000000 --- a/inference-engine/include/mkldnn/mkldnn_extension_ptr.hpp +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (C) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief A header file that defines a wrapper class for handling extension instantiation and releasing resources - * @file mkldnn_extension_ptr.hpp - */ -#pragma once - -#include "details/ie_so_pointer.hpp" -#include "mkldnn/mkldnn_extension.hpp" -#include <string> -#include <memory> - -namespace InferenceEngine { -namespace details { - -/** - * @deprecated use new extensibility API - * @brief The SOCreatorTrait class defines the name of the fabric - * for creating MKLDNNPlugin::IMKLDNNExtension object in DLL - */ -template<> -class SOCreatorTrait<MKLDNNPlugin::IMKLDNNExtension> { -public: - /** - * @brief A name of the fabric for creating an MKLDNNPlugin::IMKLDNNExtension object in DLL - */ - static constexpr auto name = "CreateMKLDNNExtension"; -}; - -} // namespace details - -namespace MKLDNNPlugin { - -/** - * @deprecated use new extensibility API - * @brief This class is a C++ helper to work with objects created using extensions. - * Implements different interfaces. - */ -class MKLDNNExtension : public MKLDNNPlugin::IMKLDNNExtension { -public: - /** - * @brief Loads extension from a shared library - * @param name Logical name of the extension library (soname without .dll/.so/lib prefix) - */ - explicit MKLDNNExtension(const std::string &name) - : actual(name) {} - - /** - * @brief Creates a generic layer and returns a pointer to an instance - * @param primitive Pointer to a newly created layer - * @param layer Layer parameters (source for name, type, precision, attr, weights...) - * @param resp Optional: pointer to an already allocated object to contain information in case of failure - * @return Status code of the operation: OK (0) for success - */ - InferenceEngine::StatusCode CreateGenericPrimitive(IMKLDNNGenericPrimitive *&primitive, - const InferenceEngine::CNNLayerPtr &layer, - InferenceEngine::ResponseDesc *resp) const noexcept override { - return actual->CreateGenericPrimitive(primitive, layer, resp); - } - - /** - * @brief This method isn't implemented for the old API - */ - InferenceEngine::StatusCode getPrimitiveTypes(char**& types, unsigned int& size, - InferenceEngine::ResponseDesc* resp) noexcept override { - return actual->getPrimitiveTypes(types, size, resp); - } - - /** - * @brief This method isn't implemented for the old API - */ - InferenceEngine::StatusCode getFactoryFor(InferenceEngine::ILayerImplFactory *&factory, - const InferenceEngine::CNNLayer *cnnLayer, - InferenceEngine::ResponseDesc *resp) noexcept override { - return actual->getFactoryFor(factory, cnnLayer, resp); - } - - /** - * @brief This method isn't implemented for the old API - */ - InferenceEngine::StatusCode getShapeInferImpl(InferenceEngine::IShapeInferImpl::Ptr& impl, const char* type, - InferenceEngine::ResponseDesc* resp) noexcept override { - return actual->getShapeInferImpl(impl, type, resp); - }; - - /** - * @brief Gets the extension version information - * @param versionInfo A pointer to version info, set by plugin - */ - void GetVersion(const InferenceEngine::Version *&versionInfo) const noexcept override { - actual->GetVersion(versionInfo); - } - - /** - * @brief Sets a log callback that is used to track what is going on inside - * @param listener Logging listener - */ - void SetLogCallback(InferenceEngine::IErrorListener &listener) noexcept override { - actual->SetLogCallback(listener); - } - - /** - * @brief Cleans the resources up - */ - void Unload() noexcept override { - actual->Unload(); - } - - /** - * @brief Does nothing since destruction is done via regular mechanism - */ - void Release() noexcept override {} - -protected: - /** - * @brief An SOPointer instance to the loaded templated object - */ - InferenceEngine::details::SOPointer<MKLDNNPlugin::IMKLDNNExtension> actual; -}; -} // namespace MKLDNNPlugin - - -/** - * @deprecated use new extensibility API - * @brief Creates a special shared_pointer wrapper for the given type from a specific shared module - * @param name Name of the shared library file - * @return shared_pointer A wrapper for the given type from a specific shared module - */ -template<> -inline std::shared_ptr<MKLDNNPlugin::IMKLDNNExtension> make_so_pointer(const std::string &name) { - return std::make_shared<MKLDNNPlugin::MKLDNNExtension>(name); -} - -} // namespace InferenceEngine diff --git a/inference-engine/include/mkldnn/mkldnn_extension_types.hpp b/inference-engine/include/mkldnn/mkldnn_extension_types.hpp deleted file mode 100644 index d4f675b13..000000000 --- a/inference-engine/include/mkldnn/mkldnn_extension_types.hpp +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (C) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief A header file for the main MKL-DNN Extension API to work with weights, and primitives in memory - * @file mkldnn_extension_types.hpp - */ -#pragma once - -#include "ie_common.h" -#include "ie_precision.hpp" - -namespace InferenceEngine { -namespace MKLDNNPlugin { - -/** - * @deprecated use new extensibility API - * @brief Defines formats from MKL-DNN which are supported in the MKL-DNN plugin of IE. - */ -enum MemoryFormat { - /** Undefined memory format, used for empty memory descriptors. */ - format_undef = 0, - /** Unspecified format. The primitive selects a format - * automatically. */ - any, - /** A tensor in a generic format described by the stride and blocking - * values in each dimension. See #mkldnn_blocking_desc_t for more - * information. */ - blocked, - /** 1D data tensor. */ - x, - /** 2D data tensor. */ - nc, - /** 4D data tensor in the @c nchw format typically used in Caffe. */ - nchw, - /** 4D data tensor in the @c nhwc format typically used in TensorFlow. */ - nhwc, - /** 4D data tensor in the @c chwn format typically used in Neon. */ - chwn, - /** 4D data tensor in the @c nchw format with channels data laid out in - * memory in 8-element blocks. */ - nChw8c, - /** 4D data tensor in the @c nchw format with channels data laid out in - * memory in 16-element blocks. */ - nChw16c, - /** 2D weights tensor in the format (input channels, output channels). */ - oi, - /** 2D weights tensor in the format (input channels, output channels). */ - io, - /** 4D weights tensor in the format (input channels, output channels, - * width, height). */ - oihw, - /** 4D weights tensor in the format (input channels, height, width, - * output channels). */ - ihwo, - /** 4D weights tensor in the format (height, width, input channels, - * output channels). */ - hwio, - /** 4D weights tensor in the @c oihw format with both input and output - * channels data laid out in memory in 8-element blocks. */ - OIhw8i8o, - /** 4D weights tensor in the @c oihw format with both input and output - * channels data laid out in memory in 16-element blocks. */ - OIhw16i16o, - /** 4D weights tensor in the @c oihw format with output channels data - * laid out in memory in 16-element blocks and input channels data - * laid out in memory in 8-element blocks blocked by pairs. */ - OIhw8i16o2i, - /** 4D weights tensor in the @c oihw format with input channels data - * laid out in memory in 16-element blocks and output channels data - * laid out in memory in 8-element blocks blocked by pairs. */ - OIhw8o16i2o, - /** 4D weights tensor in the @c oihw format with both input and output - * channels data laid out in memory in 8-element blocks. */ - OIhw8o8i, - /** 4D weights tensor in the @c oihw format with both input and output - * channels data laid out in memory in 16-element blocks. */ - OIhw16o16i, - /** 4D weights tensor in the format (output channels, input channels, - * height, width) with output channels data laid out in memory in 8-element - * blocks. */ - Oihw8o, - /** 4D weights tensor in the format (output channels, input channels, - * height, width) with output channels data laid out in memory in - * 16-element blocks. */ - Oihw16o, - /** 4D weights tensor in the format (output channels, width, height, input - * channels) with output channels data laid out in memory in 8-element - * blocks. */ - Ohwi8o, - /** 4D weights tensor in the format (output channels, width, height, input - * channels) with output channels data laid out in memory in 16-element - * blocks. */ - Ohwi16o, - /** 4D weights tensor in the @c oihw format with both input and output - * channels data laid out in memory in 16-element and 4-element blocks. */ - OhIw16o4i, - /** 5D weights tensor in the @c oihw format with extra outer dimension for - * groups. */ - goihw, - /** 5D weights tensor in the blocked version of @c goihw format with both - * input and output channels data laid out in memory in 8-element blocks. - */ - gOIhw8i8o, - /** 5D weights tensor in the blocked version of @c goihw format with both - * input and output channels data laid out in memory in 16-element blocks. - */ - gOIhw16i16o, - /** 5D weights tensor in the @c oihw format with output channels data - * laid out in memory in 16-element blocks and input channels data - * laid out in memory in 8-element blocks blocked by pairs. */ - gOIhw8i16o2i, - /** 5D weights tensor in the @c oihw format with input channels data - * laid out in memory in 16-element blocks and output channels data - * laid out in memory in 8-element blocks blocked by pairs. */ - gOIhw8o16i2o, - /** 5D weights tensor in the blocked version of @c goihw format with both - * input and output channels data laid out in memory in 8-element blocks. - */ - gOIhw8o8i, - /** 5D weights tensor in the blocked version of @c goihw format with both - * input and output channels data laid out in memory in 16-element blocks. - */ - gOIhw16o16i, - /** 5D weights tensor in the blocked version of @c goihw format with output - * channels data laid out in memory in 8-element blocks. */ - gOihw8o, - /** 5D weights tensor in the blocked version of @c goihw format with output - * channels data laid out in memory in 16-element blocks. */ - gOihw16o, - /** 5D weights tensor in the blocked version of @c goihw format with output - * channels data laid out in memory in 8-element blocks. */ - gOhwi8o, - /** 5D weights tensor in the blocked version of @c goihw format with output - * channels data laid out in memory in 16-element blocks. */ - gOhwi16o, - /** 5D weights tensor in the @c goihw format with both input and output - * channels data laid out in memory in 16-element and 4-element blocks. */ - gOhIw16o4i, - /** 4D weights tensor in the oihw format with input channels data laid out - * in memory in 8-element blocks. */ - oIhw8i = nChw8c, - /** 4D weights tensor in the oihw format with input channels data laid out - * in memory in 16-element blocks. */ - oIhw16i = nChw16c, -}; - -/** - * @deprecated use new extensibility API - * @brief Stores necessary information about the primitive memory object. - * Such as precision, dimensions, memory format etc. - */ -struct MKLDNNPrimitiveMemory { - /** - * @brief precision type - */ - Precision precision; - /** - * @brief dimensions of the given primitive - */ - SizeVector dims; - /** - * @brief memory type of the given primitive - */ - MemoryFormat format; - /** - * @brief primitive data stored - */ - void *data; - - /** - * @brief A constructor. - */ - MKLDNNPrimitiveMemory() : format(format_undef), data(nullptr) {} -}; - -/** - * @deprecated use new extensibility API - * @brief Stores necessary information about the primitive weights. - */ -struct MKLDNNWeightsMemory { - /** - * @brief size of weights - */ - size_t size; - /** - * @brief pointer to weights data - */ - void *data; - - /** - * @brief A constructor. - */ - MKLDNNWeightsMemory() : size(0), data(nullptr) {} -}; - -} // namespace MKLDNNPlugin -} // namespace InferenceEngine diff --git a/inference-engine/include/mkldnn/mkldnn_generic_primitive.hpp b/inference-engine/include/mkldnn/mkldnn_generic_primitive.hpp deleted file mode 100644 index da93c4c29..000000000 --- a/inference-engine/include/mkldnn/mkldnn_generic_primitive.hpp +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (C) 2018 Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -/** - * @brief a header file for MKL-DNN Generic Primitive API - * @file mkldnn_generic_primitive.hpp - */ -#pragma once - -#include "mkldnn_extension_types.hpp" -#include "details/ie_irelease.hpp" -#include <vector> - -namespace InferenceEngine { -namespace MKLDNNPlugin { - -/** - * @deprecated use new extensibility API - * @brief The MKLDNNGenericFormats stores weights, biases, inputs and outputs of the primitive - */ -class MKLDNNGenericFormats { -public: - /** - * @brief A default constructor - * @param ins - vector of inputs - * @param outs - vector of outputs - * @param weights - weights, format_undef by default - * @param biases - biases, format_undef by default - */ - MKLDNNGenericFormats(const std::vector<MemoryFormat> &ins, const std::vector<MemoryFormat> &outs, - const MemoryFormat weights = MemoryFormat::format_undef, - const MemoryFormat biases = MemoryFormat::format_undef) : inputs(ins), outputs(outs) { - this->weights = weights; - this->biases = biases; - } - - /** - * @brief Get input formats - * @return vector of input formats - */ - const std::vector<MemoryFormat>& GetInputs() const noexcept { - return inputs; - } - - /** - * @brief Get output formats - * @return vector of output formats - */ - const std::vector<MemoryFormat>& GetOutputs() const noexcept { - return outputs; - } - - /** - * @brief Get weights format - * @return weights format - */ - const MemoryFormat& GetWeights() const noexcept { - return weights; - } - - /** - * @brief Get biases format - * @return biases format - */ - const MemoryFormat& GetBiases() const noexcept { - return biases; - } - -private: - std::vector<MemoryFormat> inputs; - std::vector<MemoryFormat> outputs; - MemoryFormat weights; - MemoryFormat biases; -}; - -/** - * @deprecated use new extensibility API - * @brief The IMKLDNNGenericPrimitive is the main Generic Primitive interface - */ -class IMKLDNNGenericPrimitive : public InferenceEngine::details::IRelease { -public: - void Release() noexcept override { - delete this; - } - - /** - * @brief Sets inputs nd outputs - * @param inputs - vector of input primitives - * @param outputs - vector of output primitives - */ - void SetMemory(const std::vector<MKLDNNPrimitiveMemory>& inputs, - const std::vector<MKLDNNPrimitiveMemory>& outputs) noexcept { - this->inputs = inputs; - this->outputs = outputs; - } - - /** - * @brief Gets supported formats - * @return vector of supported formats - */ - virtual std::vector<MKLDNNGenericFormats> GetSupportedFormats() noexcept = 0; - - /** - * @brief Entry point of actual execution of primitive. - * Error reporting mechanism missed, static check should be done in constructor - */ - virtual void Execute() noexcept = 0; - -protected: - /** - * @brief Vector of input primitives - */ - std::vector<MKLDNNPrimitiveMemory> inputs; - /** - * @brief Vector of output primitives - */ - std::vector<MKLDNNPrimitiveMemory> outputs; -}; - -} // namespace MKLDNNPlugin -} // namespace InferenceEngine |