[android] Build qnn subplugin

- Add ndk-build/makefile to build qnn subplugin.
- Add Java enum type for qnn and fIx script to build qnn subplugin.

Signed-off-by: Yongjoo Ahn <yongjoo1.ahn@samsung.com>

6 files changed, 128 insertions, 1 deletions

diff --git a/java/android/nnstreamer/build.gradle b/java/android/nnstreamer/build.gradle
index 48c1c04..3d48420 100644
--- a/java/android/nnstreamer/build.gradle
+++ b/java/android/nnstreamer/build.gradle
@@ -107,6 +107,11 @@ android {
                 println 'Set jniLibs.srcDirs includes libraries for SNPE'
             }
 
+            if (project.hasProperty('QNN_EXT_LIBRARY_PATH')) {
+                jniLibs.srcDirs += project.properties['QNN_EXT_LIBRARY_PATH']
+                println 'Set jniLibs.srcDirs includes libraries for QNN'
+            }
+
             if (project.hasProperty('NNFW_EXT_LIBRARY_PATH')) {
                 jniLibs.srcDirs += project.properties['NNFW_EXT_LIBRARY_PATH']
                 println 'Set jniLibs.srcDirs includes libraries for NNFW'
diff --git a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java
index e3e7045..3e00db4 100644
--- a/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java
+++ b/java/android/nnstreamer/src/main/java/org/nnsuite/nnstreamer/NNStreamer.java
@@ -86,6 +86,11 @@ public final class NNStreamer {
          */
         MXNET,
         /**
+         * <a href="https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk">QNN</a>
+         * provides lower-level, unified APIs for Qualcomm AI development.
+         */
+        QNN,
+        /**
          * Unknown framework (usually error)
          */
         UNKNOWN
diff --git a/java/android/nnstreamer/src/main/jni/Android-qnn.mk b/java/android/nnstreamer/src/main/jni/Android-qnn.mk
new file mode 100644
index 0000000..5938b0d
--- /dev/null
+++ b/java/android/nnstreamer/src/main/jni/Android-qnn.mk
@@ -0,0 +1,63 @@
+#------------------------------------------------------
+# QNN
+#------------------------------------------------------
+LOCAL_PATH := $(call my-dir)
+
+ifndef NNSTREAMER_ROOT
+$(error NNSTREAMER_ROOT is not defined!)
+endif
+
+include $(NNSTREAMER_ROOT)/jni/nnstreamer.mk
+
+QNN_DIR := $(LOCAL_PATH)/qnn
+
+ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
+QNN_LIB_PATH := $(QNN_DIR)/lib
+else
+$(error Target arch ABI not supported: $(TARGET_ARCH_ABI))
+endif
+
+QNN_INCLUDES := $(QNN_DIR)/include/QNN
+
+#------------------------------------------------------
+# qnn-sdk (prebuilt shared library)
+#------------------------------------------------------
+QNN_PREBUILT_LIBS :=
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libQnnCpu
+LOCAL_SRC_FILES := $(QNN_LIB_PATH)/libQnnCpu.so
+include $(PREBUILT_SHARED_LIBRARY)
+QNN_PREBUILT_LIBS += libQnnCpu
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libQnnGpu
+LOCAL_SRC_FILES := $(QNN_LIB_PATH)/libQnnGpu.so
+include $(PREBUILT_SHARED_LIBRARY)
+QNN_PREBUILT_LIBS += libQnnGpu
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libQnnDsp
+LOCAL_SRC_FILES := $(QNN_LIB_PATH)/libQnnDsp.so
+include $(PREBUILT_SHARED_LIBRARY)
+QNN_PREBUILT_LIBS += libQnnDsp
+
+include $(CLEAR_VARS)
+LOCAL_MODULE := libQnnHtp
+LOCAL_SRC_FILES := $(QNN_LIB_PATH)/libQnnHtp.so
+include $(PREBUILT_SHARED_LIBRARY)
+QNN_PREBUILT_LIBS += libQnnHtp
+
+#------------------------------------------------------
+# tensor-filter sub-plugin for qnn
+#------------------------------------------------------
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := qnn-subplugin
+LOCAL_SRC_FILES := $(NNSTREAMER_FILTER_QNN_SRCS)
+LOCAL_CXXFLAGS := -O3 -fPIC -frtti -fexceptions $(NNS_API_FLAGS)
+LOCAL_C_INCLUDES := $(QNN_INCLUDES) $(NNSTREAMER_INCLUDES) $(GST_HEADERS_COMMON)
+LOCAL_STATIC_LIBRARIES := nnstreamer
+LOCAL_SHARED_LIBRARIES := $(QNN_PREBUILT_LIBS)
+
+include $(BUILD_STATIC_LIBRARY)
diff --git a/java/android/nnstreamer/src/main/jni/Android.mk b/java/android/nnstreamer/src/main/jni/Android.mk
index a6aaf18..33a8fe1 100644
--- a/java/android/nnstreamer/src/main/jni/Android.mk
+++ b/java/android/nnstreamer/src/main/jni/Android.mk
@@ -76,6 +76,9 @@ _ENABLE_SNPE := false
 endif #endif ($(filter $(TARGET_ARCH_ABI), arm64-v8a,)
 endif #endif ($(ENABLE_SNPE),true)
 
+# QNN
+ENABLE_QNN := false
+
 # PyTorch
 ENABLE_PYTORCH := false
 
@@ -137,6 +140,13 @@ NNS_SUBPLUGINS += snpe-subplugin
 include $(LOCAL_PATH)/Android-snpe.mk
 endif
 
+ifeq ($(ENABLE_QNN),true)
+NNS_API_FLAGS += -DENABLE_QNN=1
+NNS_SUBPLUGINS += qnn-subplugin
+
+include $(LOCAL_PATH)/Android-qnn.mk
+endif
+
 ifeq ($(ENABLE_PYTORCH),true)
 NNS_API_FLAGS += -DENABLE_PYTORCH=1
 NNS_SUBPLUGINS += pytorch-subplugin
diff --git a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c
index cc3f726..95f636f 100644
--- a/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c
+++ b/java/android/nnstreamer/src/main/jni/nnstreamer-native-api.c
@@ -155,6 +155,9 @@ extern void init_filter_nnfw (void);
 #if defined (ENABLE_SNPE)
 extern void init_filter_snpe (void);
 #endif
+#if defined (ENABLE_QNN)
+extern void init_filter_qnn (void);
+#endif
 #if defined (ENABLE_PYTORCH)
 extern void init_filter_torch (void);
 #endif
@@ -844,6 +847,9 @@ nns_get_nnfw_type (jint fw_type, ml_nnfw_type_e * nnfw)
     case 5: /* NNFWType.MXNET */
       *nnfw = ML_NNFW_TYPE_MXNET;
       break;
+    case 6: /* NNFWType.QNN */
+      *nnfw = ML_NNFW_TYPE_QNN;
+      break;
     default: /* Unknown */
       _ml_logw ("Unknown NNFW type (%d).", fw_type);
       return FALSE;
@@ -918,7 +924,7 @@ nnstreamer_native_initialize (JNIEnv * env, jobject context)
 #endif /* ENABLE_FLATBUF */
 #endif
 
-#if defined (ENABLE_SNPE) || defined (ENABLE_TFLITE_QNN_DELEGATE)
+#if defined (ENABLE_QNN) || defined (ENABLE_SNPE) || defined (ENABLE_TFLITE_QNN_DELEGATE)
     /* some filters require additional tasks */
     if (!_qc_android_set_env (env, context)) {
       _ml_logw ("Failed to set environment variables for QC Android. Some features may not work properly.");
@@ -942,6 +948,9 @@ nnstreamer_native_initialize (JNIEnv * env, jobject context)
 #if defined (ENABLE_SNPE)
     init_filter_snpe ();
 #endif
+#if defined (ENABLE_QNN)
+    init_filter_qnn ();
+#endif
 #if defined (ENABLE_PYTORCH)
     init_filter_torch ();
 #endif
diff --git a/java/build-nnstreamer-android.sh b/java/build-nnstreamer-android.sh
index 0a4df9e..7f98186 100755
--- a/java/build-nnstreamer-android.sh
+++ b/java/build-nnstreamer-android.sh
@@ -56,6 +56,9 @@
 ##@@   --enable_snpe=(yes|no)
 ##@@       'yes'      : build with sub-plugin for SNPE
 ##@@       'no'       : [default]
+##@@   --enable_qnn=(yes|no)
+##@@       'yes'      : build with sub-plugin for QNN
+##@@       'no'       : [default]
 ##@@   --enable_pytorch=(yes(:(1.10.1))?|no)
 ##@@       'yes'      : build with sub-plugin for PyTorch. You can optionally specify the version of
 ##@@                    PyTorch to use by appending ':version' [1.10.1 is the default].
@@ -117,6 +120,9 @@ enable_nnfw="yes"
 # Enable SNPE
 enable_snpe="no"
 
+# Enable QNN
+enable_qnn="no"
+
 # Enable PyTorch
 enable_pytorch="no"
 
@@ -212,6 +218,9 @@ for arg in "$@"; do
         --enable_snpe=*)
             enable_snpe=${arg#*=}
             ;;
+        --enable_qnn=*)
+            enable_qnn=${arg#*=}
+            ;;
         --enable_pytorch=*)
             IFS=':' read -ra enable_pytorch_args <<< "${arg#*=}"
             is_valid_pytorch_version=0
@@ -284,6 +293,7 @@ elif [[ $build_type == "internal" ]]; then
     enable_nnfw="yes"
     enable_nnfw_ext="yes"
     enable_snpe="no"
+    enable_qnn="no"
     enable_pytorch="no"
     enable_tflite="no"
     enable_mxnet="no"
@@ -335,6 +345,13 @@ if [[ $enable_snpe == "yes" ]]; then
     echo "Build with SNPE: $SNPE_DIRECTORY"
 fi
 
+if [[ $enable_qnn == "yes" ]]; then
+    [ -z "$QNN_DIRECTORY" ] && echo "Need to set QNN_DIRECTORY, to build sub-plugin for QNN." && exit 1
+    [ $target_abi != "arm64-v8a" ] && echo "Set target ABI arm64-v8a to build sub-plugin for QNN." && exit 1
+
+    echo "Build with QNN: $QNN_DIRECTORY"
+fi
+
 if [[ $enable_pytorch == "yes" ]]; then
     echo "Build with PyTorch $pytorch_ver"
 fi
@@ -579,6 +596,24 @@ if [[ $enable_snpe == "yes" ]]; then
     mv nnstreamer/src/main/jni/snpe/lib/ext/arm64-v8a/libSNPE.so nnstreamer/src/main/jni/snpe/lib
 fi
 
+# Update QNN option
+if [[ $enable_qnn == "yes" ]]; then
+    sed -i "s|ENABLE_QNN := false|ENABLE_QNN := true|" nnstreamer/src/main/jni/Android.mk
+    sed -i "$ a QNN_EXT_LIBRARY_PATH=src/main/jni/qnn/lib/ext" gradle.properties
+
+    mkdir -p nnstreamer/src/main/jni/qnn/lib/ext/arm64-v8a
+    cp -r $QNN_DIRECTORY/include nnstreamer/src/main/jni/qnn
+
+    # Copy libs to build QNN
+    cp $QNN_DIRECTORY/lib/aarch64-android/libQnnCpu.so nnstreamer/src/main/jni/qnn/lib/
+    cp $QNN_DIRECTORY/lib/aarch64-android/libQnnGpu.so nnstreamer/src/main/jni/qnn/lib/
+    cp $QNN_DIRECTORY/lib/aarch64-android/libQnnDsp.so nnstreamer/src/main/jni/qnn/lib/
+    cp $QNN_DIRECTORY/lib/aarch64-android/libQnnHtp.so nnstreamer/src/main/jni/qnn/lib/
+
+    # Copy external so files for QNN
+    cp $QNN_DIRECTORY/lib/hexagon-v*/unsigned/libQnn*Skel.so nnstreamer/src/main/jni/qnn/lib/ext/arm64-v8a
+fi
+
 # Update PyTorch option
 if [[ $enable_pytorch == "yes" ]]; then
     sed -i "s|ENABLE_PYTORCH := false|ENABLE_PYTORCH := true|" nnstreamer/src/main/jni/Android.mk