diff options
author | Tae-Young Chung <ty83.chung@samsung.com> | 2020-08-04 14:10:27 +0900 |
---|---|---|
committer | Tae-Young Chung <ty83.chung@samsung.com> | 2020-09-02 14:11:58 +0900 |
commit | b03beaa2d298f75bf806ac00f72ea72446aba774 (patch) | |
tree | f45c3922fbd8fb0f116810ff0825cc6f333434f5 | |
parent | 1dc4fcd3ef876a2ab95b087c1f4084e9cc31fa3c (diff) | |
download | mediavision-b03beaa2d298f75bf806ac00f72ea72446aba774.tar.gz mediavision-b03beaa2d298f75bf806ac00f72ea72446aba774.tar.bz2 mediavision-b03beaa2d298f75bf806ac00f72ea72446aba774.zip |
[WIP #1] support human pose with a camera
Signed-off-by: Tae-Young Chung <ty83.chung@samsung.com>
-rwxr-xr-x | mv_inference/inference/src/Inference.cpp | 12 | ||||
-rw-r--r-- | mv_inference/inference/src/Posture.cpp | 3 | ||||
-rw-r--r-- | test/testsuites/stream_infer/stream_infer.c | 356 |
3 files changed, 96 insertions, 275 deletions
diff --git a/mv_inference/inference/src/Inference.cpp b/mv_inference/inference/src/Inference.cpp index 29e7ae7d..75a17039 100755 --- a/mv_inference/inference/src/Inference.cpp +++ b/mv_inference/inference/src/Inference.cpp @@ -327,12 +327,15 @@ namespace inference } std::string userListName; - while (!fp.eof()) { + mUserListName.clear(); + while (fp.peek() != EOF) { std::getline(fp, userListName); + LOGI("%s", userListName.c_str()); if (userListName.length()) mUserListName.push_back(userListName); } + LOGI("size: %zd", mUserListName.size()); fp.close(); return MEDIA_VISION_ERROR_NONE; @@ -1459,9 +1462,16 @@ namespace inference int poseIndex = 0; // return error if there is no mUserListName but the number_of_landmark // is not 16 + for(std::vector<std::string>::iterator iter=mUserListName.begin(); + iter != mUserListName.end(); ++iter) { + LOGI("%s", (*iter).c_str()); + } + LOGI("number_of_pose: %d", (int)number_of_pose); for (int index = 0; index < number_of_pose; index++) { poseIndex = index; + LOGI("index: %d", index); if (!mUserListName.empty()) { + LOGI("%s", mUserListName[index].c_str()); poseIndex = std::stoi(mUserListName[index]) - 1; if (poseIndex < 0) { continue; diff --git a/mv_inference/inference/src/Posture.cpp b/mv_inference/inference/src/Posture.cpp index df21629a..05349dcd 100644 --- a/mv_inference/inference/src/Posture.cpp +++ b/mv_inference/inference/src/Posture.cpp @@ -171,6 +171,9 @@ int Posture::setPoseFromFile(const std::string motionCaptureFilePath, const std: while (!fp.eof()) { std::getline(fp, line); LOGD("%s", line.c_str()); + if (line.empty()) + continue; + std::istringstream lineStream(line); std::string token; std::vector<std::string> parsedString; diff --git a/test/testsuites/stream_infer/stream_infer.c b/test/testsuites/stream_infer/stream_infer.c index 3d3041ea..fb991e9a 100644 --- a/test/testsuites/stream_infer/stream_infer.c +++ b/test/testsuites/stream_infer/stream_infer.c @@ -126,9 +126,14 @@ static guint bus_watch_id; #define PE_TFLITE_AICLiteQ_1_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet1_0709_dq.tflite" #define PE_TFLITE_AICLiteQ_2_WEIGHT_PATH "/usr/share/capi-media-vision/models/PE/tflite/posenet2_0709_f.tflite" -static float thValNeck = 0.3f; // 15% -static float thValArm = 0.1f; // 10 % -static float thValLeg = 0.2f; // 5% +#define PLD_MOTION_CAPTURE_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example.bvh" +#define PLD_MOTION_CAPTURE_MAPPING_FILE_PATH "/usr/share/capi-media-vision/models/PLD/mocap/example-mocap-mapping.txt" + +#define POSE_RESET_COUNT 5 + +static float thPoseScore = 0.3f; +static int thResetCount = 5; +static float thCustom = 0.0f; typedef struct _rect { int left; @@ -204,6 +209,12 @@ mv_inference_h hp_mv_infer; mv_engine_config_h hp_mv_engine_cfg2; mv_inference_h hp_mv_infer2; +// pose comparison +mv_pose_h hpPoser; +float hpPoseScore; +int hpPoseHoldTime; +int hpPoseCount; + static void * outputTensorData; FILE *fp; @@ -282,6 +293,10 @@ void int_handler(int sig) free(outputTensorData); outputTensorData = NULL; } + + if (hpPoser) + mv_pose_destroy(hpPoser); + printf(TEXT_YELLOW "exit..\n" TEXT_RESET); signal(SIGINT, SIG_DFL); exit(0); @@ -393,37 +408,29 @@ void _human_pose_cb ( { printf("%d landmarks\n", number_of_pose_estimation); - float score = 0.0f; - int cnt = 0; - for (int n = 0; n < number_of_pose_estimation; ++n) { + mv_pose_compare(hpPoser, locations, + (MV_INFERENCE_HUMAN_BODY_PART_LEG_LEFT | MV_INFERENCE_HUMAN_BODY_PART_LEG_RIGHT), + &hpPoseScore); + - //humanSkeleton.pose[n].x = (int)(640.f*(float)(locations->landmarks[n].point.x + poseRoi.point.x)/192.f); - //humanSkeleton.pose[n].y = (int)(480.f*(float)(locations->landmarks[n].point.y + poseRoi.point.y)/192.f); + if (hpPoseScore >= thPoseScore) { + hpPoseHoldTime++; + } else { + if (hpPoseHoldTime > thResetCount) { + hpPoseCount++; + } + hpPoseHoldTime = 0; + } + printf("Score: %.4f, HoldTime: %d, Count: %d\n", hpPoseScore, hpPoseHoldTime, hpPoseCount); + for (int n = 0; n < number_of_pose_estimation; ++n) { humanSkeleton.pose[n].x = (int)(640.f*(float)(locations->landmarks[n].point.x)/192.f); humanSkeleton.pose[n].y = (int)(480.f*(float)(locations->landmarks[n].point.y)/192.f); humanSkeleton.scores[n] = locations->landmarks[n].score; - /* - if (score >= 0.0 ) { - score += humanSkeleton.scores[n]; - cnt++; - } - */ - //printf("(%d,%d): %f\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y, confidences[n]); - //printf("(%d,%d)\n", humanSkeleton.pose[n].x, humanSkeleton.pose[n].y); } humanSkeleton.IsDetected = true; - //score /= (float)cnt; - /* - if (score >= 0.5) { - humanSkeleton.IsDetected = true; - } else { - humanSkeleton.IsDetected = false; - humanSkeleton.label = -1; - } - */ } static gboolean @@ -561,187 +568,11 @@ draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp, cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7); cairo_set_line_width(cr, 2.0); - float poseWeight = 0.7f; float prevPoseWeight = 0.3f; if (!humanSkeleton.IsDetected) return; - /* - // - if (humanSkeleton.isPrevPose == false) { - humanSkeleton.prevPose[1] = humanSkeleton.pose[1]; - // head - neck - if (humanSkeleton.scores[0] >= thValNeck ) { - humanSkeleton.prevPose[0] = humanSkeleton.pose[0]; - } - - // right arm - if (humanSkeleton.scores[2] >= thValArm) { - // neck - right shoulder - humanSkeleton.prevPose[2] = humanSkeleton.pose[2]; - if (humanSkeleton.scores[3] >= thValArm) { - // right shoulder - right elbow - humanSkeleton.prevPose[3] = humanSkeleton.pose[3]; - if (humanSkeleton.scores[4] >= thValArm) { - // right elbow - right wrist - humanSkeleton.prevPose[4] = humanSkeleton.pose[4]; - } - } - } - - // left arm - if (humanSkeleton.scores[5] >= thValArm) { - // neck - right shoulder - humanSkeleton.prevPose[5] = humanSkeleton.pose[5]; - if (humanSkeleton.scores[6] >= thValArm) { - // right shoulder - right elbow - humanSkeleton.prevPose[6] = humanSkeleton.pose[6]; - if (humanSkeleton.scores[7] >= thValArm) { - // right elbow - right wrist - humanSkeleton.prevPose[7] = humanSkeleton.pose[7]; - } - } - } - - // right leg - if (humanSkeleton.scores[8] >= thValLeg) { - // neck - right shoulder - humanSkeleton.prevPose[8] = humanSkeleton.pose[8]; - if (humanSkeleton.scores[9] >= thValLeg) { - // right shoulder - right elbow - humanSkeleton.prevPose[9] = humanSkeleton.pose[9]; - if (humanSkeleton.scores[10] >= thValLeg) { - // right elbow - right wrist - humanSkeleton.prevPose[10] = humanSkeleton.pose[10]; - } - } - } - - // left leg - if (humanSkeleton.scores[11] >= thValLeg) { - // neck - right shoulder - humanSkeleton.prevPose[11] = humanSkeleton.pose[11]; - if (humanSkeleton.scores[12] >= thValLeg) { - // right shoulder - right elbow - humanSkeleton.prevPose[12] = humanSkeleton.pose[12]; - if (humanSkeleton.scores[13] >= thValLeg) { - // right elbow - right wrist - humanSkeleton.prevPose[13] = humanSkeleton.pose[13]; - } - } - } - humanSkeleton.isPrevPose = true; - } else { - // weighted sum of pose and prevPose - // method1: fixed weights (pose : prevPose = 0.7 : 0.3) - float poseWeight = 0.7f; - float prevPoseWeight = 0.3f; - humanSkeleton.prevPose[1].x = (poseWeight * humanSkeleton.pose[1].x + - prevPoseWeight * humanSkeleton.prevPose[1].x); - humanSkeleton.prevPose[1].y = (poseWeight * humanSkeleton.pose[1].y + - prevPoseWeight * humanSkeleton.prevPose[1].y); - // head - neck - if (humanSkeleton.scores[0] >= thValNeck ) { - humanSkeleton.prevPose[0].x = (poseWeight * humanSkeleton.pose[0].x + - prevPoseWeight * humanSkeleton.prevPose[0].x); - humanSkeleton.prevPose[0].y = (poseWeight * humanSkeleton.pose[0].y + - prevPoseWeight * humanSkeleton.prevPose[0].y); - } - - // right arm - if (humanSkeleton.scores[2] >= thValArm) { - // neck - right shoulder - humanSkeleton.prevPose[2].x = (poseWeight * humanSkeleton.pose[2].x + - prevPoseWeight * humanSkeleton.prevPose[2].x); - humanSkeleton.prevPose[2].y = (poseWeight * humanSkeleton.pose[2].y + - prevPoseWeight * humanSkeleton.prevPose[2].y); - if (humanSkeleton.scores[3] >= thValArm) { - // right shoulder - right elbow - humanSkeleton.prevPose[3].x = (poseWeight * humanSkeleton.pose[3].x + - prevPoseWeight * humanSkeleton.prevPose[3].x); - humanSkeleton.prevPose[3].y = (poseWeight * humanSkeleton.pose[3].y + - prevPoseWeight * humanSkeleton.prevPose[3].y); - if (humanSkeleton.scores[4] >= thValArm) { - // right elbow - right wrist - humanSkeleton.prevPose[4].x = (poseWeight * humanSkeleton.pose[4].x + - prevPoseWeight * humanSkeleton.prevPose[4].x); - humanSkeleton.prevPose[4].y = (poseWeight * humanSkeleton.pose[4].y + - prevPoseWeight * humanSkeleton.prevPose[4].y); - } - } - } - - // left arm - if (humanSkeleton.scores[5] >= thValArm) { - // neck - right shoulder - humanSkeleton.prevPose[5].x = (poseWeight * humanSkeleton.pose[5].x + - prevPoseWeight * humanSkeleton.prevPose[5].x); - humanSkeleton.prevPose[5].y = (poseWeight * humanSkeleton.pose[5].y + - prevPoseWeight * humanSkeleton.prevPose[5].y); - if (humanSkeleton.scores[6] >= thValArm) { - // right shoulder - right elbow - humanSkeleton.prevPose[6].x = (poseWeight * humanSkeleton.pose[6].x + - prevPoseWeight * humanSkeleton.prevPose[6].x); - humanSkeleton.prevPose[6].y = (poseWeight * humanSkeleton.pose[6].y + - prevPoseWeight * humanSkeleton.prevPose[6].y); - if (humanSkeleton.scores[7] >= thValArm) { - // right elbow - right wrist - humanSkeleton.prevPose[7].x = (poseWeight * humanSkeleton.pose[7].x + - prevPoseWeight * humanSkeleton.prevPose[7].x); - humanSkeleton.prevPose[7].y = (poseWeight * humanSkeleton.pose[7].y + - prevPoseWeight * humanSkeleton.prevPose[7].y); - } - } - } - - // right leg - if (humanSkeleton.scores[8] >= thValLeg) { - // neck - right shoulder - humanSkeleton.prevPose[8].x = (poseWeight * humanSkeleton.pose[8].x + - prevPoseWeight * humanSkeleton.prevPose[8].x); - humanSkeleton.prevPose[8].y = (poseWeight * humanSkeleton.pose[8].y + - prevPoseWeight * humanSkeleton.prevPose[8].y); - if (humanSkeleton.scores[9] >= thValLeg) { - // right shoulder - right elbow - humanSkeleton.prevPose[9].x = (poseWeight * humanSkeleton.pose[9].x + - prevPoseWeight * humanSkeleton.prevPose[9].x); - humanSkeleton.prevPose[9].y = (poseWeight * humanSkeleton.pose[9].y + - prevPoseWeight * humanSkeleton.prevPose[9].y); - if (humanSkeleton.scores[10] >= thValLeg) { - // right elbow - right wrist - humanSkeleton.prevPose[10].x = (poseWeight * humanSkeleton.pose[10].x + - prevPoseWeight * humanSkeleton.prevPose[10].x); - humanSkeleton.prevPose[10].y = (poseWeight * humanSkeleton.pose[10].y + - prevPoseWeight * humanSkeleton.prevPose[10].y); - } - } - } - - // left leg - if (humanSkeleton.scores[11] >= thValLeg) { - // neck - right shoulder - humanSkeleton.prevPose[11].x = (poseWeight * humanSkeleton.pose[11].x + - prevPoseWeight * humanSkeleton.prevPose[11].x); - humanSkeleton.prevPose[11].y = (poseWeight * humanSkeleton.pose[11].y + - prevPoseWeight * humanSkeleton.prevPose[11].y); - if (humanSkeleton.scores[12] >= thValLeg) { - // right shoulder - right elbow - humanSkeleton.prevPose[12].x = (poseWeight * humanSkeleton.pose[12].x + - prevPoseWeight * humanSkeleton.prevPose[12].x); - humanSkeleton.prevPose[12].y = (poseWeight * humanSkeleton.pose[12].y + - prevPoseWeight * humanSkeleton.prevPose[12].y); - if (humanSkeleton.scores[13] >= thValLeg) { - // right elbow - right wrist - humanSkeleton.prevPose[13].x = (poseWeight * humanSkeleton.pose[13].x + - prevPoseWeight * humanSkeleton.prevPose[13].x); - humanSkeleton.prevPose[13].y = (poseWeight * humanSkeleton.pose[13].y + - prevPoseWeight * humanSkeleton.prevPose[13].y); - } - } - } - } -*/ for (int k = 0; k < 16; ++k) { if (humanSkeleton.isPrevPose == false) { humanSkeleton.prevPose[k] = humanSkeleton.pose[k]; @@ -756,84 +587,54 @@ draw_overlay_pose (GstElement * overlay, cairo_t * cr, guint64 timestamp, for (int k = 0; k < 16; ++k) { if (humanSkeleton.scores[k] > 0.0f) { + if (k > 9) + cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7); + cairo_arc(cr, humanSkeleton.prevPose[k].x, humanSkeleton.prevPose[k].y, 3,0, 2*M_PI); cairo_stroke(cr); } } -/* - // - //draw.. - // head - neck - if (humanSkeleton.scores[0] >= thValNeck ) { - cairo_move_to(cr, humanSkeleton.prevPose[0].x, humanSkeleton.prevPose[0].y); - cairo_line_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y); - } - - // right arm - cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y); - if (humanSkeleton.scores[2] >= thValArm) { - // neck - right shoulder - cairo_line_to(cr, humanSkeleton.prevPose[2].x, humanSkeleton.prevPose[2].y); - if (humanSkeleton.scores[3] >= thValArm) { - // right shoulder - right elbow - cairo_line_to(cr, humanSkeleton.prevPose[3].x, humanSkeleton.prevPose[3].y); - if (humanSkeleton.scores[4] >= thValArm) { - // right elbow - right wrist - cairo_line_to(cr, humanSkeleton.prevPose[4].x, humanSkeleton.prevPose[4].y); - } - } - } - cairo_stroke(cr); - // left arm - cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y); - if (humanSkeleton.scores[5] >= thValArm) { - // neck - right shoulder - cairo_line_to(cr, humanSkeleton.prevPose[5].x, humanSkeleton.prevPose[5].y); - if (humanSkeleton.scores[6] >= thValArm) { - // right shoulder - right elbow - cairo_line_to(cr, humanSkeleton.prevPose[6].x, humanSkeleton.prevPose[6].y); - if (humanSkeleton.scores[7] >= thValArm) { - // right elbow - right wrist - cairo_line_to(cr, humanSkeleton.prevPose[7].x, humanSkeleton.prevPose[7].y); - } - } - } - cairo_stroke(cr); + cairo_select_font_face(cr, "sans-serif", CAIRO_FONT_SLANT_NORMAL, CAIRO_FONT_WEIGHT_BOLD); + cairo_set_font_size(cr,15); - // right leg - cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y); - if (humanSkeleton.scores[8] >= thValLeg) { - // neck - right shoulder - cairo_line_to(cr, humanSkeleton.prevPose[8].x, humanSkeleton.prevPose[8].y); - if (humanSkeleton.scores[9] >= thValLeg) { - // right shoulder - right elbow - cairo_line_to(cr, humanSkeleton.prevPose[9].x, humanSkeleton.prevPose[9].y); - if (humanSkeleton.scores[10] >= thValLeg) { - // right elbow - right wrist - cairo_line_to(cr, humanSkeleton.prevPose[10].x, humanSkeleton.prevPose[10].y); - } - } - } - cairo_stroke(cr); + char howToText[2][1024]; + char tmpText[3][1024]; + snprintf(howToText[0], 1024, "Until hold time %d over Score: %.2f", thResetCount, thPoseScore); + snprintf(howToText[1], 1024, "Green-Hold, Blue-Release"); + snprintf(tmpText[0], 1024, "Score: %.4f", hpPoseScore); + snprintf(tmpText[1], 1024, "Hold time: %d", hpPoseHoldTime > thResetCount ? thResetCount : hpPoseHoldTime); + snprintf(tmpText[2], 1024, "Count: %2d", hpPoseCount); + + + cairo_set_source_rgba(cr, 1.0, 1.0, 1.0, 0.7); + cairo_move_to(cr, 400.0, 30.0); + cairo_show_text(cr, howToText[0]); + cairo_move_to(cr, 420.0, 45.0); + cairo_show_text(cr, howToText[1]); + if (hpPoseScore >= thPoseScore) { + if (hpPoseHoldTime > thResetCount) + cairo_set_source_rgba(cr, 0.0, 0.1, 0.9, 0.7); + else + cairo_set_source_rgba(cr, 0.1, 0.9, 0.0, 0.7); + + cairo_move_to(cr, 420.0, 60.0); + cairo_show_text(cr, tmpText[0]); + cairo_move_to(cr, 420.0, 75.0); + cairo_show_text(cr, tmpText[1]); + cairo_move_to(cr, 420.0, 90.0); + cairo_show_text(cr, tmpText[2]); - // left leg - cairo_move_to(cr, humanSkeleton.prevPose[1].x, humanSkeleton.prevPose[1].y); - if (humanSkeleton.scores[11] >= thValLeg) { - // neck - right shoulder - cairo_line_to(cr, humanSkeleton.prevPose[11].x, humanSkeleton.prevPose[11].y); - if (humanSkeleton.scores[12] >= thValLeg) { - // right shoulder - right elbow - cairo_line_to(cr, humanSkeleton.prevPose[12].x, humanSkeleton.prevPose[12].y); - if (humanSkeleton.scores[13] >= thValLeg) { - // right elbow - right wrist - cairo_line_to(cr, humanSkeleton.prevPose[13].x, humanSkeleton.prevPose[13].y); - } - } + } else { + cairo_set_source_rgba(cr, 0.9, 0.1, 0.0, 0.7); + cairo_move_to(cr, 420.0, 60.0); + cairo_show_text(cr, tmpText[0]); + cairo_move_to(cr, 420.0, 75.0); + cairo_show_text(cr, tmpText[1]); + cairo_move_to(cr, 420.0, 90.0); + cairo_show_text(cr, tmpText[2]); } - cairo_stroke(cr); - */ } static void @@ -1636,6 +1437,12 @@ static int app_create(void *data) if (ad->modelType == MODEL_TYPE_POSE_CPM) { err = perform_armnn_human_pose_cpm_configure(hp_mv_engine_cfg); //err = perform_tflite_human_pose_cpm_configure(hp_mv_engine_cfg); + + mv_pose_create(&hpPoser); + mv_pose_set_from_file(hpPoser, + PLD_MOTION_CAPTURE_FILE_PATH, + PLD_MOTION_CAPTURE_MAPPING_FILE_PATH); + } else if (ad->modelType == MODEL_TYPE_POSE_HAND_AICLite) { outputTensorData = (void*)calloc(56*56*21, sizeof(float)); err = perform_tflite_hand_detection_AIC(hp_mv_engine_cfg); @@ -1711,6 +1518,7 @@ static int app_create(void *data) queue4 = gst_element_factory_make("queue", "queue4"); queue5 = gst_element_factory_make("queue", "queue5"); encconv = gst_element_factory_make("videoconvert", "encconv"); + g_object_set(G_OBJECT(enc), "bitrate", 800000, NULL); } @@ -2002,7 +1810,7 @@ int main (int argc, char *argv[]) } if (argc < 2) { - printf("usage: mv_stream_infer model [NeckThresVal, ArmThresVal, LegThresVal, [filename]]"); + printf("usage: mv_stream_infer model [thPoseScore, thResetCount, thCustom, [filename]]"); printf("model: 0(CPM), 1(AIC Hand), 2(AIC Lite Hand), 3(AIC Lite Q Hand)\n"); return -1; } @@ -2016,9 +1824,9 @@ int main (int argc, char *argv[]) if (ad.modelType != MODEL_TYPE_POSE_HAND_AICLite && ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2 && ad.modelType != MODEL_TYPE_POSE_HAND_AICLite2Q) { - thValNeck = (float)atoi(argv[2])/100.f; - thValArm = (float)atoi(argv[3])/100.f; - thValLeg = (float)atoi(argv[4])/100.f; + thPoseScore = (float)atoi(argv[2])/100.f; + thResetCount = atoi(argv[3]); + thCustom = (float)atoi(argv[4])/100.f; poseRoi.point.x = 50; poseRoi.point.y = 0; |