From 74402be3ca494b843364cac17773c9a8fa420f75 Mon Sep 17 00:00:00 2001 From: Alexander Biryukov Date: Mon, 4 Jun 2018 08:24:21 +0500 Subject: [PATCH] Smooth landmarks on significant face position or pose changes --- game/main_code.cpp | 23 +++++++------------ game/main_code.h | 4 ++-- .../FaceLandmarkDetector.cpp | 7 +++--- .../DoubleHitBalls-win/FaceLandmarkDetector.h | 4 ++-- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/game/main_code.cpp b/game/main_code.cpp index 1556dc8..6ed18b1 100755 --- a/game/main_code.cpp +++ b/game/main_code.cpp @@ -11,8 +11,6 @@ //#define USE_PREDICTION -#define SMOOTH_LANDMARKS - cv::Point flipVertical(cv::Point point) { return cv::Point(point.x, 720 - point.y); } @@ -53,21 +51,19 @@ FaceStruct::FaceStruct(const std::array& from ApplyPreds(fromPreds, 0.0, cameraMatrix, distortionCoefficients); } -void FaceStruct::CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) +void FaceStruct::CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) { -#ifdef SMOOTH_LANDMARKS for(size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { - if(similarity < 0.01) { + if(noticeablyChanged) { + landmarkSmoothers[i * 2].reset(); + landmarkSmoothers[i * 2 + 1].reset(); + } else { preds[i] = Vector2f( landmarkSmoothers[i * 2].responsiveAnalogReadSimple(preds[i](0)), landmarkSmoothers[i * 2 + 1].responsiveAnalogReadSimple(preds[i](1)) ); - } else { - landmarkSmoothers[i * 2].reset(); - landmarkSmoothers[i * 2 + 1].reset(); } } -#endif float minX = preds[0](0); float maxX = preds[0](0); @@ -150,10 +146,10 @@ void FaceStruct::CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat size = {maxX- minX, maxY - minY}; } -void FaceStruct::ApplyPreds(const std::array& fromPreds, double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) +void FaceStruct::ApplyPreds(const std::array& fromPreds, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients) { InnerApplyPreds(fromPreds); - CalcFromPreds(similarity, cameraMatrix, distortionCoefficients); + CalcFromPreds(noticeablyChanged, cameraMatrix, distortionCoefficients); } void FaceStruct::InnerApplyPreds(const std::array& fromPreds) @@ -491,10 +487,7 @@ void TMyApplication::InnerUpdate(size_t dt) { if (faceLandmarkArr[index].valid) { - faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr, faceLandmarkArr[index].similarity, cameraMatrix, distortionCoefficients); - std::ostringstream similarityText; - similarityText << std::setprecision(3) << faceLandmarkArr[index].similarity; - cv::putText(renderImage, similarityText.str(), cv::Point(50, index * 60 + 60), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(255, 255, 255)); + faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr, faceLandmarkArr[index].noticeablyChanged, cameraMatrix, distortionCoefficients); for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { diff --git a/game/main_code.h b/game/main_code.h index 0a03e74..29cc6d8 100755 --- a/game/main_code.h +++ b/game/main_code.h @@ -52,8 +52,8 @@ struct FaceStruct FaceStruct(); FaceStruct(const std::array& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients); - void CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients); - void ApplyPreds(const std::array& fromPred, double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficientss); + void CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients); + void ApplyPreds(const std::array& fromPred, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficientss); void InnerApplyPreds(const std::array& fromPreds); }; diff --git a/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.cpp b/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.cpp index 6c96e1b..b47bd7d 100755 --- a/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.cpp +++ b/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.cpp @@ -248,7 +248,7 @@ void FaceLandmarkDetector::FindLandmarkThreadProc(size_t threadIndex) localOutputLandmarks.faceRect = localInputFace.faceRect; localOutputLandmarks.frameIndex = localInputFace.frameIndex; - localOutputLandmarks.similarity = localInputFace.similarity; + localOutputLandmarks.noticeablyChanged = localInputFace.noticeablyChanged; localOutputLandmarks.valid = true; } else @@ -450,10 +450,11 @@ void FaceLandmarkDetector::InnerSeparateFacesToBuckets(std::vector& fa } } - inOutBucketArr[j].similarity = calcSimilarity(inOutBucketArr[j].faceFrame, facesBelongsToSpace[j][lastIndex].second); inOutBucketArr[j].faceRect = facesBelongsToSpace[j][lastIndex].first; inOutBucketArr[j].faceFrame = facesBelongsToSpace[j][lastIndex].second; - + inOutBucketArr[j].noticeablyChanged = + calcSimilarity(inOutBucketArr[j].faceFrame, facesBelongsToSpace[j][lastIndex].second) > 0.01 || + lastDistance > 100.0; facesBelongsToSpace[j].erase(facesBelongsToSpace[j].begin() + lastIndex); diff --git a/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h b/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h index f4d1766..24b53eb 100755 --- a/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h +++ b/windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h @@ -35,7 +35,7 @@ struct FaceIntermediateStruct cv::Mat faceFrame; int frameIndex; - double similarity = 0.0; + bool noticeablyChanged = false; }; @@ -47,7 +47,7 @@ struct FaceLandmarkStruct std::array landmarkArr; int frameIndex; - double similarity = 0.0; + bool noticeablyChanged = false; };