// Copyright (C) 2018 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // #pragma once #include #include #include #include #include #include #include #include #include "details/caseless.hpp" #include "mkldnn_dims.h" #include "mkldnn_memory.h" #include "mkldnn_edge.h" #include "mkldnn_descriptor.h" #include "mkldnn/iml_type_mapper.h" #include "mkldnn_extension_mngr.h" #include "mkldnn_primitive.h" namespace MKLDNNPlugin { using MKLDNNNodePtr = std::shared_ptr; using MKLDNNNodeWeakPtr = std::weak_ptr; enum Type { Unknown, Generic, Reorder, Input, Output, Convolution, Deconvolution, Convolution_Sum, Convolution_Activation, Convolution_Depthwise, Convolution_Sum_Activation, Activation, Depthwise, Lrn, Pooling, FullyConnected, SoftMax, Split, Concatenation, Power, Eltwise, Gemm, Crop, Reshape, Tile, SimplerNMS, ROIPooling, BatchNormalization, Flatten, Permute, Copy, MemoryOutput, MemoryInput, LSTMCell, RNN }; static Type TypeFromName(const std::string type) { static InferenceEngine::details::caseless_unordered_map type_to_name_tbl = { { "Unknown", Unknown }, { "Input", Input }, { "Const", Input }, { "Output", Output }, { "Reorder", Reorder }, { "Convolution", Convolution }, { "ReLU", Activation }, { "ELU", Activation }, { "Sigmoid", Activation }, { "Logistic", Activation }, { "TanH", Activation }, { "ReLU6", Activation }, { "Activation", Activation }, { "ScaleShift", Depthwise }, { "PReLU", Depthwise }, { "Clamp", Activation }, { "Norm", Lrn }, { "LRN", Lrn }, { "Pooling", Pooling }, { "FullyConnected", FullyConnected }, { "InnerProduct", FullyConnected }, { "Gemm", Gemm }, { "Softmax", SoftMax }, { "SoftMax", SoftMax }, { "Split", Split }, { "Slice", Split }, { "Concat", Concatenation }, { "Power", Power }, { "Deconvolution", Deconvolution }, { "Eltwise", Eltwise }, { "Crop", Crop }, { "Reshape", Reshape }, { "Tile", Tile }, { "SimplerNMS", SimplerNMS }, { "ROIPooling", ROIPooling }, { "BatchNormalization", BatchNormalization }, { "Flatten", Flatten }, { "Permute", Permute }, { "Copy", Copy }, { "LSTMCell", LSTMCell }, { "RNN", RNN }, { "MemoryInput", MemoryInput}, // for construction from name ctor, arbitrary name is used { "Memory", MemoryOutput }, // for construction from layer ctor }; if (type_to_name_tbl.find(type) != type_to_name_tbl.end()) { return type_to_name_tbl[type]; } else { return Unknown; } } class PrimitiveDescInfo { public: PrimitiveDescInfo(const InferenceEngine::LayerConfig conf, impl_desc_type type): config(conf) { implementationType = type; } PrimitiveDescInfo(const PrimitiveDescInfo &descInfo) = default; PrimitiveDescInfo(PrimitiveDescInfo &&descInfo) = default; PrimitiveDescInfo &operator=(const PrimitiveDescInfo &descInfo) = default; const InferenceEngine::LayerConfig getConfig() const { return config; } InferenceEngine::LayerConfig& getConfig() { return config; } impl_desc_type getImplementationType() const { return implementationType; } private: InferenceEngine::LayerConfig config; impl_desc_type implementationType; }; class MKLDNNNode : public InferenceEngine::details::no_copy { public: static MKLDNNNode* CreateNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng, const MKLDNNExtensionManager::Ptr& extMgr); ~MKLDNNNode() override = default; void addEdge(const MKLDNNEdgeWeakPtr& edge, size_t pIndex, size_t cIndex, bool insertChildIndex = false); void removeEdge(const MKLDNNEdgeWeakPtr& edge); virtual void cleanup(); void remove(); const std::vector &getParentEdges() const noexcept { return parentEdges; } const std::vector &getChildEdges() const noexcept { return childEdges; } const MKLDNNEdgePtr getParentEdgeAt(size_t idx) const; virtual const MKLDNNEdgePtr getChildEdgeAt(size_t idx) const; bool isDropped() { return (isEdgesEmpty(childEdges) && isEdgesEmpty(parentEdges)); } const mkldnn::engine& getEngine() const { return engine; } bool isConstant(); bool isInplace() const; void fuseWith(const MKLDNNNodePtr &fuse) { fusedWith.push_back(fuse); } void mergeWith(const MKLDNNNodePtr &merge) { mergedWith.push_back(merge); } const std::vector &getMergeWith() { return mergedWith; } const std::vector &getFusedWith() { return fusedWith; } const std::string getName() const { return name; } Type getType() const { return type; } const InferenceEngine::CNNLayerPtr &getCnnLayer() const { return cnnLayer; } const std::vector& getSupportedPrimitiveDescriptors() const { return supportedPrimitiveDescriptors; } inline const PrimitiveDescInfo* getSelectedPrimitiveDescriptor() const { if (selectedPrimitiveDescriptorIndex < 0 || selectedPrimitiveDescriptorIndex >= supportedPrimitiveDescriptors.size()) return nullptr; return &supportedPrimitiveDescriptors[selectedPrimitiveDescriptorIndex]; } inline PrimitiveDescInfo* getSelectedPrimitiveDescriptor() { if (selectedPrimitiveDescriptorIndex < 0 || selectedPrimitiveDescriptorIndex >= supportedPrimitiveDescriptors.size()) return nullptr; return &supportedPrimitiveDescriptors[selectedPrimitiveDescriptorIndex]; } void selectPrimitiveDescriptorByIndex(int index) { if (index < 0 || index >= supportedPrimitiveDescriptors.size()) selectedPrimitiveDescriptorIndex = -1; else selectedPrimitiveDescriptorIndex = index; } std::string getPrimitiveDescriptorType(); PerfCount &PerfCounter() { return perfCounter; } virtual void setDynamicBatchLim(int lim); void resolveNotAllocatedEdges(); virtual void execute(mkldnn::stream strm); virtual void initSupportedPrimitiveDescriptors(); virtual void createPrimitive() = 0; virtual void selectOptimalPrimitiveDescriptor(); virtual void initOptimalPrimitiveDescriptor(); virtual void getSupportedDescriptors() = 0; virtual void createDescriptor(const std::vector& inputDesc, const std::vector& outputDesc) {} virtual void initDescriptor(const InferenceEngine::LayerConfig& config); virtual bool created() const = 0; virtual bool created(const MKLDNNExtensionManager::Ptr& extMgr) { return created(); } template PD createPrimitiveDescriptor(const mkldnn::primitive_attr &attr = mkldnn::primitive_attr()) { auto descsEqual = [](const std::vector& srcDescs, const std::vector& selectedDescs) { if (srcDescs.empty() && selectedDescs.empty()) return true; if (srcDescs.empty() || selectedDescs.empty()) return false; for (size_t i = 0; i < srcDescs.size() && i < selectedDescs.size(); i++) { if (srcDescs[i] != selectedDescs[i].desc && srcDescs[i].getLayout() != InferenceEngine::Layout::ANY) return false; } return true; }; const PrimitiveDescInfo *selected_pd = getSelectedPrimitiveDescriptor(); if (selected_pd == nullptr) THROW_IE_EXCEPTION << "Preferable primitive descriptor does not set for node " << getName() << "."; for (const auto& desc : descs) { try { mkldnn::primitive_desc_iterator itpd = desc.createPrimitiveDescriptorIterator(engine, attr); do { std::vector srcDescs; for (size_t i = 0; i < desc.inputNumbers(); i++) srcDescs.push_back(getSrcMemDesc(itpd, i)); std::vector dstDescs; for (size_t i = 0; i < desc.outputNumbers(); i++) dstDescs.push_back(getDstMemDesc(itpd, i)); impl_desc_type impl_type = parse_impl_name(itpd.get_impl_info_str()); if (impl_type == selected_pd->getImplementationType() && descsEqual(srcDescs, selected_pd->getConfig().inConfs) && descsEqual(dstDescs, selected_pd->getConfig().outConfs)) { prepareMemory(selected_pd, itpd); PD prim_desc = createPd(desc); itpd.getPrimitiveDescriptor(prim_desc); return prim_desc; } } while (itpd.next()); } catch (std::exception& e) { // it throw exception in case of no implementation found continue; } } THROW_IE_EXCEPTION << "Primitive descriptor was not found for node " << getName() << "."; } static void invertVectorCopyUtoI(const InferenceEngine::PropertyVector& src, std::vector& dst) { dst.clear(); for (int i = 1; i <= src.size(); i++) { dst.push_back(static_cast(src[src.size() - i])); } } protected: // TODO: It is necessary only in order to avoid modifications of cnnLayers and original topology std::vector outDims; std::vector inDims; void setType(Type type) { this->type = type; } virtual int getMaxBatch(); virtual InferenceEngine::TensorDesc getConfiguredInputDesc(const InferenceEngine::LayerConfig& config, size_t idx) const; virtual InferenceEngine::TensorDesc getConfiguredOutputDesc(const InferenceEngine::LayerConfig& config, size_t idx) const; virtual MKLDNNMemoryDesc getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx); virtual MKLDNNMemoryDesc getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx); typedef std::function GetPrimitiveMemoryFormatFunc; std::vector internalBlobDesc; std::vector fusedWith; std::vector mergedWith; std::vector implPriorities; MKLDNNNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng); int selectedPrimitiveDescriptorIndex = -1; bool permanent = false; bool temporary = false; int dynBatchLim = 0; enum class ConstantType { Unknown, Const, NoConst }; ConstantType constant = ConstantType::Unknown; std::vector internalBlobs; std::vector internalBlobMemory; std::vector supportedPrimitiveDescriptors; MKLDNNPrimitive prim; std::vector descs; InferenceEngine::Blob::Ptr ext_scales; friend class MKLDNNEdge; friend class MKLDNNGraph; friend class MKLDNNGraphOptimizer; bool isUninitTensorDesc(const InferenceEngine::TensorDesc& desc) const; bool isInitConfig(const InferenceEngine::LayerConfig& config) const; virtual void selectPreferPrimitiveDescriptor(const std::vector& priority); virtual bool canBeInPlace() const; virtual const std::vector& getPrimitivesPriority(); std::vector getAvailableFormatsForDims(const MKLDNNDims& dims) const; int batchToProcess(); InferenceEngine::Blob::Ptr createInternalBlob(InferenceEngine::SizeVector dims, bool weights); template class Register { public: Register() { Registry::RegisterNode( Registry::CreatorByLayerFunction( [](const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng) -> MKLDNNNode* { return new To(layer, eng); } ) ); } }; private: std::vector parentEdges; std::vector childEdges; InferenceEngine::CNNLayerPtr cnnLayer; mkldnn::engine engine; std::string name; const std::string typeStr; Type type; int execIndex = -1; std::string typeToStr(Type type); PerfCount perfCounter; InferenceEngine::ProfilingTask profilingTask; bool isEdgesEmpty(const std::vector& edges) const; class Registry { public: typedef std::function CreatorByLayerFunction; static MKLDNNNode *CreateNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng, const MKLDNNExtensionManager::Ptr& extMgr); static void RegisterNode(CreatorByLayerFunction f); private: static std::vector _dataByLayer; }; template typename std::enable_if::value, PD>::type createPd(MKLDNNDescriptor desc) { std::shared_ptr selected_desc_ptr = desc; std::shared_ptr backward_prim_desc_ptr = desc; return PD(*selected_desc_ptr, engine, *backward_prim_desc_ptr); } template typename std::enable_if::value, PD>::type createPd(MKLDNNDescriptor desc) { std::shared_ptr selected_desc_ptr = desc; return PD(*selected_desc_ptr, engine); } void prepareMemory(const PrimitiveDescInfo *selected_pd, mkldnn::primitive_desc_iterator& itpd); enum LOOK { LOOK_UP = 1, LOOK_DOWN = 2 }; ConstantType checkConstant(LOOK look, std::vector& checkNodes); }; template inline T div_up(const T a, const U b) { assert(b); return (a + b - 1) / b; } template inline T rnd_up(const T a, const U b) { return div_up(a, b) * b; } } // namespace MKLDNNPlugin