From ec0d5090fce2fd1a65fa43a87898c364f170fa35 Mon Sep 17 00:00:00 2001
From: Alexander Biryukov <alexander.biryukov@fishrungames.com>
Date: Fri, 1 Jun 2018 23:09:53 +0500
Subject: [PATCH] Landmark smoothers

---
 game/main_code.cpp                            | 36 +++++++++++++------
 game/main_code.h                              |  7 +---
 .../DoubleHitBalls-win/ValueSmoother.h        |  4 +--
 3 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/game/main_code.cpp b/game/main_code.cpp
index f3b4623..1556dc8 100755
--- a/game/main_code.cpp
+++ b/game/main_code.cpp
@@ -11,6 +11,8 @@
 
 //#define USE_PREDICTION
 
+#define SMOOTH_LANDMARKS
+
 cv::Point flipVertical(cv::Point point) {
 	return cv::Point(point.x, 720 - point.y);
 }
@@ -53,6 +55,20 @@ FaceStruct::FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& from
 
 void FaceStruct::CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
 {
+#ifdef SMOOTH_LANDMARKS
+	for(size_t i = 0; i < LANDMARK_POINT_COUNT; i++) {
+		if(similarity < 0.01) {
+			preds[i] = Vector2f(
+				landmarkSmoothers[i * 2].responsiveAnalogReadSimple(preds[i](0)),
+				landmarkSmoothers[i * 2 + 1].responsiveAnalogReadSimple(preds[i](1))
+			);
+		} else {
+			landmarkSmoothers[i * 2].reset();
+			landmarkSmoothers[i * 2 + 1].reset();
+		}
+	}
+#endif
+
 	float minX = preds[0](0);
 	float maxX = preds[0](0);
 	float minY = preds[0](1);
@@ -107,17 +123,17 @@ void FaceStruct::CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat
 
 	cv::solvePnP(modelPoints, imagePoints, cameraMatrix, distortionCoefficients, rotationVector, translationVector);
 
-	if(similarity > 0.01) {
-		for (size_t i = 0; i < 3; i++) {
-			rotationVectorSmoothers[i].reset();
-			translationVectorSmoothers[i].reset();
-		}
-	}
+	//if(similarity > 0.01) {
+	//	for (size_t i = 0; i < 3; i++) {
+	//		rotationVectorSmoothers[i].reset();
+	//		translationVectorSmoothers[i].reset();
+	//	}
+	//}
 
-	for(size_t i = 0; i < 3; i++) {
+	/*for(size_t i = 0; i < 3; i++) {
 		rotationVector.at<double>(i) = rotationVectorSmoothers[i].responsiveAnalogReadSimple(rotationVector.at<double>(i));
-		// translationVector.at<double>(i) = translationVectorSmoothers[i].responsiveAnalogReadSimple(translationVector.at<double>(i));
-	}
+		 translationVector.at<double>(i) = translationVectorSmoothers[i].responsiveAnalogReadSimple(translationVector.at<double>(i));
+	}*/
 
 	cv::projectPoints(boxModelSrc, rotationVector, translationVector, cameraMatrix, distortionCoefficients, boxModelDst);
 
@@ -478,7 +494,7 @@ void TMyApplication::InnerUpdate(size_t dt)
 			faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr, faceLandmarkArr[index].similarity, cameraMatrix, distortionCoefficients);
 			std::ostringstream similarityText;
 			similarityText << std::setprecision(3) << faceLandmarkArr[index].similarity;
-			cv::putText(renderImage, similarityText.str(), cv::Point(50, index * 60 + 60), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0, 0, 0));
+			cv::putText(renderImage, similarityText.str(), cv::Point(50, index * 60 + 60), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(255, 255, 255));
 
 			for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 			{
diff --git a/game/main_code.h b/game/main_code.h
index 49b6222..0a03e74 100755
--- a/game/main_code.h
+++ b/game/main_code.h
@@ -45,20 +45,15 @@ struct FaceStruct
 	std::vector<cv::Point3d> boxModelSrc;
 	std::vector<cv::Point2d> boxModelDst;
 
+	ValueSmoother landmarkSmoothers[2 * LANDMARK_POINT_COUNT];
 	ValueSmoother rotationVectorSmoothers[3];
 	ValueSmoother translationVectorSmoothers[3];
 
-	std::vector<cv::Point3f> noseEndPoint3d;
-	std::vector<cv::Point2f> noseEndPoint2d;
-
 	FaceStruct();
-
 	FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
 
 	void CalcFromPreds(double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
-
 	void ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPred, double similarity, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficientss);
-
 	void InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds);
 };
 
diff --git a/windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h b/windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h
index 9770407..3082717 100755
--- a/windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h
+++ b/windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h
@@ -13,9 +13,9 @@ public:
 protected:
 	bool inited = false;
 
-	//float SNAP_MULTIPLIER_SIMPLE = 1.0 / 60;
+	float SNAP_MULTIPLIER_SIMPLE = 1.0 / 60;
 
-	float SNAP_MULTIPLIER_SIMPLE = 0.3;
+	//float SNAP_MULTIPLIER_SIMPLE = 0.3;
 
 	float smoothValue = 0;