Smooth landmarks on significant face position or pose changes

Landmark smoothers
Similarity
2018-06-04 08:24:42 +05:00 · 2018-06-01 23:09:53 +05:00 · 2018-06-01 22:31:55 +05:00 · 2018-06-01 15:42:10 +05:00 · 2018-05-25 21:41:26 +05:00 · 2017-12-11 18:00:43 +05:00
108 changed files with 4663 additions and 2781 deletions
@@ -73,6 +73,5 @@ iOSInjectionProject/
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/log.txt
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/progress.txt
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/resources
 /windows/DoubleHitBalls-win/**/*.user
 *.db
 *.opendb
@@ -1,19 +0,0 @@
 precision mediump float;
 uniform sampler2D Texture;
 uniform vec2 samplesOffset[4];
 varying vec2 texCoord;
 void main() 
 {
 	vec3 color = vec3(0, 0, 0);
 	for (int i = 0; i < 4; ++i)
 	{
 		color += texture2D(Texture, texCoord + samplesOffset[i]).rgb;
 	}
 	color /= 4;
 	//gl_FragColor = vec4(texture2D(Texture, texCoord ), 1.0);
 	gl_FragColor = vec4(color, 1.0);
 }
@@ -1,11 +0,0 @@
 attribute vec3 vPosition;
 attribute vec2 vTexCoord;
 varying vec2 texCoord;
 uniform mat4 ProjectionMatrix;
 void main() 
 {
 	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
 	texCoord = vTexCoord;
 }
@@ -1,15 +0,0 @@
 precision mediump float;
 uniform sampler2D Texture;
 uniform float Transparency;
 uniform vec4 BrickColor;
 varying vec2 texCoord;
 void main() 
 {
 	vec4 color = BrickColor * texture2D(Texture,texCoord).rgba;
 	gl_FragColor = vec4(color.rgb, color.a * Transparency);
 }
@@ -1,12 +0,0 @@
 attribute vec3 vPosition;
 attribute vec2 vTexCoord;
 varying vec2 texCoord;
 uniform mat4 ProjectionMatrix;
 void main() 
 {
 	//480x320
 	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
 	texCoord = vTexCoord;
 }
@@ -1,7 +0,0 @@
 precision mediump float;
 varying vec4 color;
 void main() 
 {
 	gl_FragColor = color;
 }
@@ -1,11 +0,0 @@
 attribute vec3 vPosition;
 attribute vec4 vColor;
 varying vec4 color;
 uniform mat4 ProjectionMatrix;
 void main() 
 {
 	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
 	color = vColor;
 }
@@ -0,0 +1,249 @@
 {
 	"transform version": "V1",
 	"models": [
 	{
 		"type": "EmojiModel",
 		"name": "Smile"
 	}],
  "renderParams": [
    {
      "vertexShader": "./assets/shaders/vertex/orthoVertexShader.txt",
      "fragmentShader": "./assets/shaders/fragment/orthoFragmentShader.txt",
      "alphaTest": "true",
      "blend": "true",
      "TAG": "smile & tongue 0"
    }
  ],
 	"initialModules": [0, 1, 2, 3, 4],
  "modules": [
    {
      "iconPath": "./assets/fx/emoji/Asset 3@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 3@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 24@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 24@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 13@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 13@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 23@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 23@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 19@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 19@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 20@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 20@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 17@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 17@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 18@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 18@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 15@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 15@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 16@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 16@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 21@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 21@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 4@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 4@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 22@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 22@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 14@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 14@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 11@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 11@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 25@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 25@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 26@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 26@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 27@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 27@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 28@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 28@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 5@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 5@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 6@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 6@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 7@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 7@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 8@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 8@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 9@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 9@20x.png"
        }
      ]
    },
    {
      "iconPath": "./assets/fx/emoji/Asset 10@20x.png",
      "suits": [
        {
          "renderParamsID": 0,
          "emojiTexture": "./assets/fx/emoji/Asset 10@20x.png"
        }
      ]
    }
  ]
 }
@@ -1,52 +0,0 @@
 precision mediump float;
 varying vec4 color;
 varying vec3 position;
 //uniform vec4 lineParams;
 //uniform vec4 startPointEndPoint;
 varying vec3 lineParams;
 varying vec4 startPointEndPoint;
 void main() 
 {
 	float a = lineParams.x;
 	float b = lineParams.y;
 	float c = lineParams.z;
 	float Xa = startPointEndPoint.r;
 	float Ya = startPointEndPoint.g;
 	float Xb = startPointEndPoint.b;
 	float Yb = startPointEndPoint.a;
 	float distance;
 	float distanceToA = (Xa - position.x)*(Xb - Xa) + (Ya - position.y)*(Yb - Ya);
 	float distanceToB = (Xb - position.x)*(Xb - Xa) + (Yb - position.y)*(Yb - Ya);
 	if (distanceToA * distanceToB > 0)
 	{
 		float len = sqrt((Xb - Xa)*(Xb - Xa) + (Yb - Ya)*(Yb - Ya));
 		distance = max(abs(distanceToA/len), abs(distanceToB/len));
 	}
 	else
 	{
 		distance = abs(a*position.x + b*position.y + c);
 	}
 	float t = clamp(1.2 / (distance + 1.0) - 0.5, 0.0, 1.0);
 	gl_FragColor = vec4(color.rgb, color.a*t);
 	//gl_FragColor = vec4(color.rgb*clamp(1.2 / (distance + 1.0) - 0.5, 0.0, 1.0), 1.0);
 }
@@ -1,22 +0,0 @@
 attribute vec3 vPosition;
 attribute vec4 vColor;
 attribute vec3 vLineParams;
 attribute vec4 vStartPointEndPoint;
 varying vec3 position;
 varying vec4 color;
 varying vec3 lineParams;
 varying vec4 startPointEndPoint;
 uniform mat4 ProjectionMatrix;
 void main() 
 {
 	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
 	position = vPosition;
 	color = vColor;
 	lineParams = vLineParams;
 	startPointEndPoint = vStartPointEndPoint;
 }
@@ -7,7 +7,7 @@ void main()
 {
 	vec4 color = texture2D(Texture,texCoord).rgba;
-	gl_FragColor = vec4(color.rgb, 1.0);
+	gl_FragColor = vec4(color.rgb, color.a * Transparency);
@@ -1,83 +0,0 @@
 #include "creditscode.h"
 #include "include/Engine.h"
 #include "main_code.h"
 const int TGameCredits::CONST_STATE_APPEARING = 0;
 const int TGameCredits::CONST_STATE_APPEARED = 1;
 const int TGameCredits::CONST_STATE_DISAPPEARING = 2;
 const int TGameCredits::CONST_STATE_DISAPPEARED = 3;
 const float TGameCredits::CONST_APPEAR_TIME = 150.f;
 TGameCredits::TGameCredits()
    : TGameAreaAncestor()
    , StateTimer(0)
    , State(CONST_STATE_DISAPPEARED)
 {
 }
 void TGameCredits::Draw()
 {
 	//*SE::Console << "TGameCredits::Draw";
    float transparency = StateTimer/CONST_APPEAR_TIME;
    RenderUniform1f("Transparency", transparency);
 	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList["main_menu_bkg_left"]);
 	Renderer->DrawRect(Vector2f(0, 0.f), Vector2f(480.f, 320.f));
 	const Vector2f creditsPos(140.f, 130.f);
 	const Vector2f creditsHalfSize(128.f, 128.f);
 	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_CREDITS_TEXTURE]);
 	Renderer->DrawRect(creditsPos-creditsHalfSize, creditsPos+creditsHalfSize);
 	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOGO_SMALL_TEXTURE]);
 	Renderer->DrawRect(Vector2f(240.f-128.f+15.f, 320.f-64.f), Vector2f(240.f+128.f+15.f, 320.f));
    CheckGlError("Draw TGameCredits");
 }
 void TGameCredits::Update(size_t dt)
 {
    if (State == CONST_STATE_APPEARING)
 	{
 		*SE::Console << "TGameCredits::Update - CONST_STATE_APPEARING";
        StateTimer += dt;
        if (StateTimer >= CONST_APPEAR_TIME)
        {
            StateTimer = CONST_APPEAR_TIME;
            State = CONST_STATE_APPEARED;
        }
    }
    else if (State == CONST_STATE_DISAPPEARING)
 	{
        StateTimer -= dt;
        if (StateTimer <= 0.f)
        {
            StateTimer = 0.f;
            State = CONST_STATE_DISAPPEARED;
        }
    }
 }
 void TGameCredits::OnTapDown(Vector2f p)
 {
 	*SE::Console << "TGameCredits::OnTapDown";
    Application->GoFromCreditsToMenu();
 }
 void TGameCredits::StartAppear()
 {
 	*SE::Console << "TGameCredits::StartAppear";
    State = CONST_STATE_APPEARING;
 }
 void TGameCredits::StartDisappear()
 {
 	*SE::Console << "TGameCredits::StartDisappear";
    State = CONST_STATE_DISAPPEARING;
 }
@@ -1,34 +0,0 @@
 #ifndef CREDITSCODE_H_INCLUDED
 #define CREDITSCODE_H_INCLUDED
 #include "game_area_interface.h"
 using namespace SE;
 class TGameCredits : public TGameAreaAncestor
 {
    static const int CONST_STATE_APPEARING;
    static const int CONST_STATE_APPEARED;
    static const int CONST_STATE_DISAPPEARING;
    static const int CONST_STATE_DISAPPEARED;
    static const float CONST_APPEAR_TIME;
    int State;
    float StateTimer;
 public:
    TGameCredits();
    virtual void Draw();
    virtual void Update(size_t dt);
    virtual void OnTapDown(Vector2f p);
    void StartAppear();
    void StartDisappear();
 };
 #endif
@@ -1,25 +0,0 @@
 #ifndef GAME_AREA_INTERFACE_H_INCLUDED
 #define GAME_AREA_INTERFACE_H_INCLUDED
 #include "include/Engine.h"
 using namespace SE;
 class TGameAreaAncestor
 {
 public:
    virtual void Draw() { }
    virtual void Update(size_t) { }
    virtual void OnTapUp(Vector2f) { }
    virtual void OnTapUpAfterMove(Vector2f) { }
    virtual void OnTapDown(Vector2f) { }
    virtual void OnFling(Vector2f) { }
 	virtual void OnScroll(Vector2f) { }
 	virtual ~TGameAreaAncestor() { } //By default destructor is not virtual
 };
 #endif
@@ -1,205 +0,0 @@
 #ifndef GAMECODE_H_INCLUDED
 #define GAMECODE_H_INCLUDED
 #include "include/Engine.h"
 #include "game_area_interface.h"
 #include <list>
 using namespace SE;
 const int CONST_BRICKSTATE_VISIBLE = 3;
 const int CONST_BRICKSTATE_DISAPPEAR = 2;
 const int CONST_BRICKSTATE_INVISIBLE = 1;
 const int CONST_BRICKSTATE_APPEAR = 0;
 const float CONST_BRICK_DISAPPEAR_TIME = 200.f;
 const float CONST_BRICK_APPEAR_TIME = 200.f;
 const int CONST_BRICKMATRIX_WIDTH = 12;
 const int CONST_BRICKMATRIX_HEIGHT = 13;
 const float CONST_BRICK_WIDTH = 37.5f;
 const float CONST_BRICK_HEIGHT = 0.5f*CONST_BRICK_WIDTH;
 const float CONST_BRICK_SHIFT_X = 15.f;
 const float CONST_BRICK_SHIFT_Y = -16.f;
 const float CONST_BONUS_APPEAR_TIME = 400.f;
 const float CONST_BONUS_FALL_SPEED = 100.f;
 const float CONST_BONUS_GOTHROUGH_TIME = 8000.f;
 const float CONST_BONUS_CATCH_DISTANCE_X = 65.f;
 const float CONST_BONUS_CATCH_DISTANCE_Y = 20.f;
 const float CONST_BONUS_FLOOR_TIME = 8000.f;
 const float CONST_BONUS_FLOOR_APPEAR_SPEED = 50.f;
 const int CONST_BONUS_TYPE_MULTIPLIER = 0;
 const int CONST_BONUS_TYPE_GOTHROUGH = 1;
 const int CONST_BONUS_TYPE_FLOOR = 2;
 class TBrick
 {
 protected:
    int InitialLocked;
    int Locked; //0, 1 or 2
    int State;
    float StateTimer;
    Vector4f Color;
 public:
    TBrick();
    void SetVisible(Vector4f color, int locked);
    void SetInvisible();
    void TryDrawAppear(int ipos, int jpos);
    void Update(size_t dt);
    Vector2f GetPosFrom(int ipos, int jpos);
    Vector4f GetColor();
    void Appear(Vector4f color, int locked);
    void Appear();
    void Hit();
    void Disappear();
    int GetLocked();
    bool CanReact();
 };
 class TBonusFalling
 {
 protected:
   float Lifetime;
   Vector2f Pos;
   std::string TexName;
   int BonusType; //0 - multiplier,  1 - Go-through, 2 - floor
 public:
    TBonusFalling(Vector2f pos);
    Vector2f GetPos();
    int GetType();
    void Draw();
    void Update(size_t dt);           
 };
 struct TBall
 {
    Vector2f Pos;
    Vector2f Velocity;
    Vector4f Color;
    std::list<Vector2f> TalePos;
    Vector2f BallInBlock;
    Vector2f PrevBallInBlock;
    TBall(Vector2f pos, Vector2f velocity, Vector4f color);
    Vector2f GetPos();
    Vector2f GetVelocityVector();
    void Go();
    void ReflectToLeft();
    void ReflectToRight();
    void ReflectToUp();
    void ReflectToDown();
    void TryReflectOnReflector(Vector2f refPos);
    void Update(size_t dt);    
 };
 typedef std::pair<Vector4f, std::string> PairColorTexture;
 struct TBlockInstansingList
 {
    //color-texture  ---> and list of triangles for this color
    std::list<std::pair<PairColorTexture, TTriangleList>> ColorBlockList;
 };
 struct TBallInstancingList
 {
    std::map<int, TTriangleList> BallAndGlowList;
 };
 class TGameLevel : public TGameAreaAncestor
 {
 protected:
    std::string BkgTexture;
    std::string LevelScreenTexture;
    std::string LevelFileName;
    Vector2f ReflectorPos;
    int LevelState;
    bool PrevLevelStateIsStandby;
    float StateTimer;
    TBrick BlockMatrix[CONST_BRICKMATRIX_WIDTH][CONST_BRICKMATRIX_HEIGHT];
    TBlockInstansingList BlockInstansingList;
    bool TapInBackBtnArea(const Vector2f& pos);
    void ReloadBlockInstansingList();
    void SetFinished();
    void SetFinishFreeze();
    void DrawBuffer();
    void DrawPauseButtons();
    void DrawBallInstancingList();
    bool RenderBufferReady;
    float OutScale;
    float OutScaleVelocity;
    void TryGoToMenu();
    std::list<TBonusFalling> BonusFallingList;
    std::list<TBall> BallList;
    TBallInstancingList BallInstancingList;
    Vector4f BallColor;
    void ReloadBallInstancingList();
    void RefreshBallInstancingList();
    void UpdateBallList(size_t dt);
    void MultiplyBalls(Vector2f pos, Vector2f velocity);
    Vector2f GetBlock(const Vector2f& pos);
    void InitLevel();
    float BonusGothroughTimer;
    float BonusFloorTimer;
    float BonusFloorPosY;
    Vector4f ParseColor(const std::string& s);
    void ReloadLevel();
 public:
    TGameLevel();
    ~TGameLevel();
    void FillWithFile(const std::string& filename);
    void SetStandBy();
    void SetLoading(const std::string& bkg, const std::string& levelscreen);
    bool IsLoaded();
 	virtual void Draw();
 	void SetPause();
 	bool IsPaused();
 	void ReleasePause();
 	virtual void Update(size_t dt);
 	virtual void OnTapDown(Vector2f pos);
 	virtual void OnTapUp(Vector2f pos);
 	virtual void OnFling(Vector2f slideSpeed);
 	virtual void OnScroll(Vector2f shift);
 };
 #endif
@@ -1,20 +0,0 @@
 #include "loadingcode.h"
 #include "include/Engine.h"
 #include "main_code.h"
 void TGameLoading::Draw()
 {
 	//*SE::Console << "TGameLoading::Draw";
 	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
 	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
    glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOADING_TEXTURE]);
 	Renderer->DrawRect(Vector2f(240.f - 128.f, 10.f), Vector2f(240.f + 128.f, 10.f + 64.f));
 	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOGO_SMALL_TEXTURE]);
 	Renderer->DrawRect(Vector2f(480.f - 256.f+50.f, 320.f - 64.f), Vector2f(480.f+50.f, 320.f));
    CheckGlError();
 }
@@ -1,15 +0,0 @@
 #ifndef LOADINGCODE_H_INCLUDED
 #define LOADINGCODE_H_INCLUDED
 #include "game_area_interface.h"
 using namespace SE;
 class TGameLoading : public TGameAreaAncestor
 {
 public:
    virtual void Draw();
 };
 #endif
@@ -0,0 +1,592 @@
 #include "main_code.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
 #include <time.h>
 #include "include/Engine.h"
 #include "main_code.h"
 FaceStruct::FaceStruct()
 {
 }
 FaceStruct::FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
 {
 	ApplyPreds(fromPreds);
 }
 void FaceStruct::CalcFromPreds()
 {
 	float minX = preds[0](0);
 	float maxX = preds[0](0);
 	float minY = preds[0](1);
 	float maxY = preds[0](1);
 	float sumX = preds[0](0);
 	float sumY = preds[0](1);
 	for (size_t i = 1; i < LANDMARK_POINT_COUNT; i++)
 	{
 		if (minX > preds[i](0))
 		{
 			minX = preds[i](0);
 		}
 		if (minY > preds[i](1))
 		{
 			minY = preds[i](1);
 		}
 		if (maxX < preds[i](0))
 		{
 			maxX = preds[i](0);
 		}
 		if (maxY < preds[i](1))
 		{
 			maxY = preds[i](1);
 		}
 		sumX += preds[i](0);
 		sumY += preds[i](1);
 	}
 	float avgX = sumX / static_cast<float>(LANDMARK_POINT_COUNT);
 	float avgY = sumY / static_cast<float>(LANDMARK_POINT_COUNT);
 	center = { avgX, avgY };
 	size = { maxX- minX,maxY - minY };
 	//center = { centerXSmoother.responsiveAnalogReadSimple(avgX), centerYSmoother.responsiveAnalogReadSimple(avgY) };
 	//size = { sizeWidthSmoother.responsiveAnalogReadSimple(maxX- minX), sizeHeightSmoother.responsiveAnalogReadSimple(maxY - minY) };
 }
 void FaceStruct::ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
 {
 	InnerApplyPreds(fromPreds);
 	CalcFromPreds();
 	/*
 	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 	{
 		preds[i] = preds[i] - center;
 	}
 	historicalPreds.push_back(preds);
 	if (historicalPreds.size() > 4)
 	{
 		historicalPreds.erase(historicalPreds.begin());
 	}
 	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 	{
 		preds[i] = { 0.f, 0.f };
 		for (size_t x = 0; x < historicalPreds.size(); x++)
 		{
 			preds[i] += historicalPreds[x][i];
 		}
 		preds[i] = preds[i] / static_cast<float>(historicalPreds.size());
 	}
 	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 	{
 		preds[i] = preds[i] + center;
 	}*/
 }
 void FaceStruct::InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
 {
 	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 	{
 		preds[i] = { fromPreds[i].x, 720 - fromPreds[i].y};
 	}	
 }
 cv::VideoCapture cap;
 FaceLandmarkDetector faceLandmarkDetector;
 GLuint lastFrameTexture;
 std::array<FaceLandmarkStruct, MAX_FACE_COUNT> faceLandmarkArr;
 cv::Mat frame;
 std::vector<cv::Mat> frameArr;
 int currentIndex = -1;
 int lastProcessedIndex = 0;
 int lineShift = 0;
 int prevPassedIndex = 0;
 int indexStep = 0;
 std::array<FaceStruct, MAX_FACE_COUNT> faceStruct;
 const std::string CONST_LOADING_BACKGROUND_BLACK = "loading_background_black";
 TMyApplication* Application;
 void TMyApplication::InnerInit()
 {
    Application = this;
 #ifdef TARGET_WIN32
 #ifdef NDEBUG
 	//ST::PathToResources = "resources/";
 	ST::PathToResources = "../../../assets/";
 #else
 	ST::PathToResources = "../../../assets/";
 #endif
 #endif
 #ifdef TARGET_IOS
    ST::PathToResources = "assets/";
 #endif
    if (Console != NULL)
    {
        *Console<<"APP INIT\n";
    }
    srand (static_cast<size_t>(time(NULL)));
 	ResourceManager->ShaderManager.AddShader("DefaultShader", "shader1vertex.txt", "shader1fragment.txt");
 	ResourceManager->ShaderManager.AddShader("FrameShader", "frameshader_vertex.txt", "frameshader_fragment.txt");
 	Renderer->PushShader("DefaultShader");
 	ResourceManager->TexList.AddTexture(CONST_LOADING_BACKGROUND_BLACK + ".png", CONST_LOADING_BACKGROUND_BLACK);
 	auto texid = ResourceManager->TexList.AddTexture("emoji/Asset 3@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 4@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 5@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 6@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 7@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 8@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 9@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 10@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 11@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 13@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 14@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 15@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 16@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 17@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 18@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 19@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 20@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 21@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 22@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 23@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 24@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 25@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 26@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 27@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 28@20x.png");
 	ResourceManager->FontManager.AddFont("droid_sans14", "droid_sans14_font_bitmap.png", "droid_sans14_font_charmap.txt");
 	ResourceManager->FontManager.PushFont("droid_sans14");
 	lastFrameTexture = ResourceManager->TexList.AddEmptyTexture("lastFrameTexture", 1280, 720);
 	Renderer->SetOrthoProjection();
 	Renderer->SetFullScreenViewport();
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		faceRenderPair[i].first.SamplerMap[CONST_STRING_TEXTURE_UNIFORM] = "Asset "+boost::lexical_cast<std::string>(i+3)+"@20x.png";
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 0, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
 		faceRenderPair[i].second.RefreshBuffer();
 	}
 	cap = cv::VideoCapture(0);
 	//cap = cv::VideoCapture("video.mp4");
 	//cap = cv::VideoCapture("bp.mp4");
 	// Check if camera opened successfully
 	if (!cap.isOpened()) {
 		std::cout << "Error opening video stream or file" << std::endl;
 	}
 	cap.set(CV_CAP_PROP_FRAME_WIDTH, 1280);
 	cap.set(CV_CAP_PROP_FRAME_HEIGHT, 720);
 	faceLandmarkDetector.StartTrackProcess();
 	Inited = true;
 }
 void TMyApplication::InnerDeinit()
 {
 	faceLandmarkDetector.StopTrackProcess();
 	cap.release();
    Inited = false;
    if (Console != NULL)
    {
        *Console<<"APP DEINIT\n";
    }
 }
 void TMyApplication::InnerOnTapDown(Vector2f p)
 {
 }
 void TMyApplication::InnerOnTapUp(Vector2f p)
 {
 }
 void TMyApplication::InnerOnTapUpAfterMove(Vector2f p)
 {
 }
 void TMyApplication::InnerOnMove(Vector2f p, Vector2f shift)
 {
 }
 void TMyApplication::OnFling(Vector2f v) 
 {
 }
 void TMyApplication::InnerDraw()
 {
 	glDisable(GL_DEPTH_TEST);
 	//glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
 	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
 	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
 	/*
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		TRenderParamsSetter params(faceRenderPair[i].first);
 		Renderer->DrawTriangleList(faceRenderPair[i].second);
 	}*/
 }
 void TMyApplication::InnerUpdate(size_t dt)
 {
 	/*cap >> lastFrame;
 	if (lastFrame.empty())
 	{
 		return;
 	}
 	faceLandmarkTracker.HandleOneFrame(lastFrame, localInputStructArr, localFaceShift, localOutputStructArr);
 	high_resolution_clock::time_point t2 = high_resolution_clock::now();
 	auto duration = duration_cast<microseconds>(t2 - FaceLandmarkTracker::lastTimePoint).count();
 	FaceLandmarkTracker::lastTimePoint = t2;
 	std::stringstream ofs;
 	ofs << round(1000000.0 / duration);
 	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
 	{
 		if (localOutputStructArr[index].valid && localInputStructArr[index].valid)
 		{
 			cv::Rect oldFaceRect = localOutputStructArr[index].face;
 			cv::Rect newFaceRect = localInputStructArr[index].face;
 			cv::Point2f oldCenter(oldFaceRect.x + oldFaceRect.width*0.5f, oldFaceRect.y + oldFaceRect.height*0.5f);
 			cv::Point2f newCenter(newFaceRect.x + newFaceRect.width*0.5f, newFaceRect.y + newFaceRect.height*0.5f);
 			localFaceShift[index] = newCenter - oldCenter;
 		}
 		else
 		{
 			localFaceShift[index] = cv::Point2f(0, 0);
 		}
 	}
 	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
 	{
 		if (localOutputStructArr[index].valid)
 		{
 			faceStruct[index].ApplyPreds(localOutputStructArr[index].preds);
 			for (size_t i = 0; i < CONST_PREDS_COUNT; i++)
 			{
 				//cv::ellipse(lastFrame, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)) + localFaceShift[index], cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
 			}
 			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.25f;
 			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.25f;
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 		else
 		{
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], {0,0}, { 0,0 }, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 	}
 	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
 	{
 		if (localInputStructArr[index].valid)
 		{
 			//cv::rectangle(lastFrame, localInputStructArr[index].face, cv::Scalar(255, 0, 255), 4, 8, 0);
 		}
 	}
 	*/
 	cap >> frame; // get a new frame from camera/video or read image
 	if (frame.empty())
 	{
 		return;
 	}
 	cv::Mat image = frame.clone();
 	frameArr.push_back(image);
 	while (frameArr.size() > 40)
 	{
 		lineShift++;
 		frameArr.erase(frameArr.begin());
 	}
 	currentIndex++;
 	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
 	bool newProcessedIndexFound = false;
 	int newProcessedIndex;
 	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
 	{
 		if (faceLandmarkArr[i].valid)
 		{
 			if (!newProcessedIndexFound)
 			{
 				newProcessedIndexFound = true;
 				newProcessedIndex = faceLandmarkArr[i].frameIndex;
 			}
 			else
 			{
 				if (newProcessedIndex < faceLandmarkArr[i].frameIndex)
 				{
 					newProcessedIndex = faceLandmarkArr[i].frameIndex;
 				}
 			}
 		}
 	}
 	if (newProcessedIndexFound && newProcessedIndex > lastProcessedIndex)
 	{
 		indexStep = 0;
 		lastProcessedIndex = newProcessedIndex;
 		prevPassedIndex = currentIndex;
 	}
 	else
 	{
 		if (prevPassedIndex > lastProcessedIndex + indexStep)
 		{
 			indexStep++;
 		}
 	}
 	//std::cout << currentIndex << " " << lastProcessedIndex << " stepped:" << lastProcessedIndex + indexStep << " indexStep: " << indexStep << std::endl;
 	cv::Mat renderImage;
 	if (lastProcessedIndex + indexStep - lineShift < 0)
 	{
 		renderImage = frameArr[0];
 	}
 	else if (lastProcessedIndex + indexStep - lineShift >= frameArr.size())
 	{
 		renderImage = frameArr[frameArr.size() - 1];
 	}
 	else
 	{
 		renderImage = frameArr[lastProcessedIndex + indexStep - lineShift];
 	}
 	renderImage = renderImage.clone();
 	cv::Rect frameRect(cv::Point(), renderImage.size());
 	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
 	{
 		if (faceLandmarkArr[i].valid)
 		{
 			cv::rectangle(renderImage, faceLandmarkArr[i].faceRect, cv::Scalar(0, 0, 255), 1, 4, 0);
 			for (size_t j = 0; j < faceLandmarkArr[i].landmarkArr.size(); j++)
 			{
 				//cv::circle(renderImage, faceLandmarkArr[i].landmarkArr[j], 0.1, cv::Scalar(0, 255, 255), 4, 8, 0);
 			}
 		}
 	}
 	for (size_t index = 0; index < faceLandmarkArr.size(); index++)
 	{
 		if (faceLandmarkArr[index].valid)
 		{
 			faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr);
 			for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 			{
 				cv::ellipse(renderImage, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)), cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
 			}
 			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.35f;
 			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.35f;
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 		else
 		{
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], { 0,0 }, { 0,0 }, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 	}
 	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
 	static std::array<char, 1280 * 720 * 3> buffer;
 	int step = renderImage.step;
 	int height = renderImage.rows;
 	int width = renderImage.cols;
 	int channels = 3;
 	char * data = (char *)renderImage.data;
 	for (int i = 0; i < height; i++)
 	{
 		int ci = height - i - 1;
 		memcpy(&buffer[i*width*channels], &(data[ci*step]), width*channels);
 	}
 	glTexImage2D(
 		GL_TEXTURE_2D,
 		0,
 		GL_RGB,
 		1280,
 		720,
 		0,
 		GL_BGR_EXT,
 		GL_UNSIGNED_BYTE,
 		&buffer[0]);
 }
 bool TMyApplication::IsInited()
 {
    return Inited;
 }
@@ -1,216 +1,561 @@
-#include "main_code.h"
+#include "main_code.h"
-
+
-#include <stdio.h>
+#include <stdio.h>
-#include <stdlib.h>
+#include <stdlib.h>
-#include <math.h>
+#include <math.h>
-#include <time.h>
+#include <time.h>
-
+
-#include "include/Engine.h"
+#include "include/Engine.h"
-
+
-#include "main_code.h"
+#include "main_code.h"
-
+
-TMyApplication* Application;
+//#define USE_PREDICTION
-
+
-static const float BUFFER_DISTANCE = 50.f;
+cv::Point flipVertical(cv::Point point) {
-
+	return cv::Point(point.x, 720 - point.y);
-
+}
-TRenderPair linesToRenderBoxes(const std::vector<std::pair<Vector2f, Vector2f>>& lineArr)
+
-{
+FaceStruct::FaceStruct()
-	TRenderPair pair;
+{
-
+	modelPoints.push_back(cv::Point3d(6.825897, 6.760612, 4.402142));     // left brow left corner
-	for (size_t i = 0; i < lineArr.size(); i++)
+	modelPoints.push_back(cv::Point3d(1.330353, 7.122144, 6.903745));     // left brow right corner
-	{
+	modelPoints.push_back(cv::Point3d(-1.330353, 7.122144, 6.903745));    // right brow left corner
-
+	modelPoints.push_back(cv::Point3d(-6.825897, 6.760612, 4.402142));    // right brow right corner
-		Vector2f startPoint = lineArr[i].first;
+	modelPoints.push_back(cv::Point3d(5.311432, 5.485328, 3.987654));     // left eye left corner
-		Vector2f endPoint = lineArr[i].second;
+	modelPoints.push_back(cv::Point3d(1.789930, 5.393625, 4.413414));     // left eye right corner
-
+	modelPoints.push_back(cv::Point3d(-1.789930, 5.393625, 4.413414));    // right eye left corner
-		Vector2f renderAreaStart{ std::min<float>(startPoint(0), endPoint(0)) - BUFFER_DISTANCE, std::min<float>(startPoint(1), endPoint(1)) - BUFFER_DISTANCE };
+	modelPoints.push_back(cv::Point3d(-5.311432, 5.485328, 3.987654));    // right eye right corner
-
+	modelPoints.push_back(cv::Point3d(2.005628, 1.409845, 6.165652));     // nose left corner
-		Vector2f renderAreEnd{ std::max<float>(startPoint(0), endPoint(0)) + BUFFER_DISTANCE, std::max<float>(startPoint(1), endPoint(1)) + BUFFER_DISTANCE };
+	modelPoints.push_back(cv::Point3d(-2.005628, 1.409845, 6.165652));    // nose right corner
-
+	modelPoints.push_back(cv::Point3d(2.774015, -2.080775, 5.048531));    // mouth left corner
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreaStart(1), 0));
+	modelPoints.push_back(cv::Point3d(-2.774015, -2.080775, 5.048531));   // mouth right corner
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreEnd(1), 0));
+	modelPoints.push_back(cv::Point3d(0.000000, -3.116408, 6.097667));    // mouth central bottom corner
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreEnd(1), 0));
+	modelPoints.push_back(cv::Point3d(0.000000, -7.415691, 4.070434));    // chin corner
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreaStart(1), 0));
+
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreEnd(1), 0));
+	poseMatrix = cv::Mat(3, 4, CV_64FC1);
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreaStart(1), 0));
+	eulerAngles = cv::Mat(3, 1, CV_64FC1);
-
+
-
+	boxModelSrc.push_back(cv::Point3d(10.0, 10.0, 10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(10.0, 10.0, -10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(10.0, -10.0, -10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(10.0, -10.0, 10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(-10.0, 10.0, 10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(-10.0, 10.0, -10.0));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
+	boxModelSrc.push_back(cv::Point3d(-10.0, -10.0, -10.0));
-
+	boxModelSrc.push_back(cv::Point3d(-10.0, -10.0, 10.0));
-		Vector2f lineVec = endPoint - startPoint;
+}
-		Vector2f lineNormVec = { -lineVec(1), lineVec(0) };
+
-
+FaceStruct::FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
-		lineNormVec.normalize();
+{
-
+	FaceStruct();
-		//float distance = abs(normVec.x*(position.x - startPoint.x) + normVec.y*(position.y - startPoint.y));
+	ApplyPreds(fromPreds, 0.0, cameraMatrix, distortionCoefficients);
-		//distance = a*x + b*y + c, here we calculate a, b,c and pass them to shader
+}
-		float a = lineNormVec(0);
+
-		float b = lineNormVec(1);
+void FaceStruct::CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
-		float c = -(lineNormVec(0)*startPoint(0) + lineNormVec(1)*startPoint(1));
+{
-
+	for(size_t i = 0; i < LANDMARK_POINT_COUNT; i++) {
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+		if(noticeablyChanged) {
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+			landmarkSmoothers[i * 2].reset();
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+			landmarkSmoothers[i * 2 + 1].reset();
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+		} else {
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+			preds[i] = Vector2f(
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
+				landmarkSmoothers[i * 2].responsiveAnalogReadSimple(preds[i](0)),
-		
+				landmarkSmoothers[i * 2 + 1].responsiveAnalogReadSimple(preds[i](1))
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+			);
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+		}
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+	}
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+	float minX = preds[0](0);
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
+	float maxX = preds[0](0);
-		
+	float minY = preds[0](1);
-		
+	float maxY = preds[0](1);
-	}
+	float sumX = preds[0](0);
-
+	float sumY = preds[0](1);
-	pair.second.RefreshBuffer();
+
-	pair.first = TRenderParams().withShader("AntialiasingShader");
+	for (size_t i = 1; i < LANDMARK_POINT_COUNT; i++)
-
+	{
-	return pair;
+		if (minX > preds[i](0))
-}
+		{
-
+			minX = preds[i](0);
-
+		}
-
+
-void TMyApplication::InnerInit()
+		if (minY > preds[i](1))
-{
+		{
-
+			minY = preds[i](1);
-    Application = this;
+		}
-    
+
-#ifdef TARGET_WIN32
+		if (maxX < preds[i](0))
-#ifdef NDEBUG
+		{
-	//ST::PathToResources = "resources/";
+			maxX = preds[i](0);
-	ST::PathToResources = "../../../assets/";
+		}
-#else
+
-	ST::PathToResources = "../../../assets/";
+		if (maxY < preds[i](1))
-#endif
+		{
-#endif
+			maxY = preds[i](1);
-    
+		}
-#ifdef TARGET_IOS
+
-    ST::PathToResources = "assets/";
+		sumX += preds[i](0);
-#endif
+		sumY += preds[i](1);
-
+	}
-    if (Console != NULL)
+
-    {
+	float avgX = sumX / static_cast<float>(LANDMARK_POINT_COUNT);
-        *Console<<"APP INIT\n";
+	float avgY = sumY / static_cast<float>(LANDMARK_POINT_COUNT);
-    }
+
-    srand (static_cast<size_t>(time(NULL)));
+	std::vector<cv::Point2f> imagePoints;
-
+	imagePoints.push_back(cv::Point2f(preds[17](0), preds[17](1))); // #17 left brow left corner
-	ResourceManager->ShaderManager.AddShader("AntialiasingShader", "ms_vertex.txt", "ms_fragment.txt");
+	imagePoints.push_back(cv::Point2f(preds[21](0), preds[21](1))); // #21 left brow right corner
-
+	imagePoints.push_back(cv::Point2f(preds[22](0), preds[22](1))); // #22 right brow left corner
-	
+	imagePoints.push_back(cv::Point2f(preds[26](0), preds[26](1))); // #26 right brow right corner
-	ResourceManager->TexList.AddTexture("console_bkg.bmp");
+	imagePoints.push_back(cv::Point2f(preds[36](0), preds[36](1))); // #36 left eye left corner
-
+	imagePoints.push_back(cv::Point2f(preds[39](0), preds[39](1))); // #39 left eye right corner
-	std::vector<std::pair<Vector2f, Vector2f>> lineArr;
+	imagePoints.push_back(cv::Point2f(preds[42](0), preds[42](1))); // #42 right eye left corner
-
+	imagePoints.push_back(cv::Point2f(preds[45](0), preds[45](1))); // #45 right eye right corner
-	for (int i = 0; i < 10; i++)
+	imagePoints.push_back(cv::Point2f(preds[31](0), preds[31](1))); // #31 nose left corner
-	{
+	imagePoints.push_back(cv::Point2f(preds[35](0), preds[35](1))); // #35 nose right corner
-		lineArr.push_back({ {50.f + i * 20.f, 50.f}, { 60.f + i * 20.f, 440.f } });
+	imagePoints.push_back(cv::Point2f(preds[48](0), preds[48](1))); // #48 mouth left corner
-
+	imagePoints.push_back(cv::Point2f(preds[54](0), preds[54](1))); // #54 mouth right corner
-		lineArr.push_back({ { 60.f + i * 20.f, 440.f },{ 70.f + i * 20.f, 50.f } });
+	imagePoints.push_back(cv::Point2f(preds[57](0), preds[57](1))); // #57 mouth central bottom corner
-	};
+	imagePoints.push_back(cv::Point2f(preds[8](0), preds[8](1)));   // #8 chin corner
-
+
-
+	cv::solvePnP(modelPoints, imagePoints, cameraMatrix, distortionCoefficients, rotationVector, translationVector);
-
+
-	lineRenderPair = linesToRenderBoxes(lineArr);
+	//if(similarity > 0.01) {
-
+	//	for (size_t i = 0; i < 3; i++) {
-	Renderer->PushShader("AntialiasingShader");
+	//		rotationVectorSmoothers[i].reset();
-
+	//		translationVectorSmoothers[i].reset();
-	Renderer->SetOrthoProjection();
+	//	}
-
+	//}
-	Renderer->SetFullScreenViewport();
+
-	
+	/*for(size_t i = 0; i < 3; i++) {
-	Inited = true;
+		rotationVector.at<double>(i) = rotationVectorSmoothers[i].responsiveAnalogReadSimple(rotationVector.at<double>(i));
-
+		 translationVector.at<double>(i) = translationVectorSmoothers[i].responsiveAnalogReadSimple(translationVector.at<double>(i));
-
+	}*/
-
+
-
+	cv::projectPoints(boxModelSrc, rotationVector, translationVector, cameraMatrix, distortionCoefficients, boxModelDst);
-
+
-}
+	// calc Euler angles
-
+	cv::Rodrigues(rotationVector, rotationMatrix);
-void TMyApplication::InnerDeinit()
+	cv::hconcat(rotationMatrix, translationVector, poseMatrix);
-{
+
-    Inited = false;
+	cv::Mat outIntrinsics = cv::Mat(3, 3, CV_64FC1);
-    Loaded = false;
+	cv::Mat outRotation = cv::Mat(3, 3, CV_64FC1);
-    if (Console != NULL)
+	cv::Mat outTranslation = cv::Mat(3, 1, CV_64FC1);
-    {
+	cv::decomposeProjectionMatrix(poseMatrix, outIntrinsics, outRotation, outTranslation, cv::noArray(), cv::noArray(), cv::noArray(), eulerAngles);
-        *Console<<"APP DEINIT\n";
+
-    }
+	center = {avgX, avgY};
-}
+	size = {maxX- minX, maxY - minY};
-
+}
-void TMyApplication::OnMouseWheel(short int delta)
+
-{
+void FaceStruct::ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
-	scale += (delta > 0 ? 1 : -1) * 0.25;
+{
-	if(scale < 0.25) {
+	InnerApplyPreds(fromPreds);
-		scale = 0.25;
+	CalcFromPreds(noticeablyChanged, cameraMatrix, distortionCoefficients);
-	}
+}
-}
+
-
+void FaceStruct::InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
-void TMyApplication::InnerOnTapDown(Vector2f p)
+{
-{
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
-
+	{
-}
+		preds[i] = { fromPreds[i].x, 720 - fromPreds[i].y};
-
+		
-void TMyApplication::InnerOnTapUp(Vector2f p)
+	}	
-{
+}
-
+
-}
+cv::VideoCapture cap;
-
+
-void TMyApplication::InnerOnTapUpAfterMove(Vector2f p)
+FaceLandmarkDetector faceLandmarkDetector;
-{
+
-
+GLuint lastFrameTexture;
-}
+
-
+std::array<FaceLandmarkStruct, MAX_FACE_COUNT> faceLandmarkArr;
-void TMyApplication::InnerOnMove(Vector2f p, Vector2f shift)
+
-{
+cv::Mat frame;
-
+
-}
+std::vector<cv::Mat> frameArr;
-	
+
-void TMyApplication::OnFling(Vector2f v) 
+int currentIndex = -1;
-{
+int lastProcessedIndex = 0;
-}
+int lineShift = 0;
-
+
-	
+int prevPassedIndex = 0;
-void TMyApplication::InnerDraw()
+
-{
+int indexStep = 0;
-
+
-	//Render the frame
+std::array<FaceStruct, MAX_FACE_COUNT> faceStruct;
-	Renderer->SetProjectionMatrix(512.f, 512.f);
+
-	Renderer->PushMatrix();
+
-	Renderer->ScaleMatrix(scale);
+const std::string CONST_LOADING_BACKGROUND_BLACK = "loading_background_black";
-
+
-	glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
+
-	glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
+TMyApplication* Application;
-	CheckGlError("");
+
-
+
-	TRenderParamsSetter params(lineRenderPair.first);
+void TMyApplication::InnerInit()
-	Renderer->DrawTriangleList(lineRenderPair.second);
+{
-
+
-	Renderer->PopMatrix();
+    Application = this;
-	CheckGlError("");
+    
-
+#ifdef TARGET_WIN32
-	
+#ifdef NDEBUG
-}
+	//ST::PathToResources = "resources/";
-
+	ST::PathToResources = "../../../assets/";
-
+#else
-void TMyApplication::InnerUpdate(size_t dt)
+	ST::PathToResources = "../../../assets/";
-{
+#endif
-
+#endif
-}
+    
-
+#ifdef TARGET_IOS
-bool TMyApplication::IsLoaded()
+    ST::PathToResources = "assets/";
-{
+#endif
-    return Loaded;
+
-}
+    if (Console != NULL)
-
+    {
-bool TMyApplication::IsInited()
+        *Console<<"APP INIT\n";
-{
+    }
-    return Inited;
+    srand (static_cast<size_t>(time(NULL)));
-}
+
 	ResourceManager->ShaderManager.AddShader("DefaultShader", "shader1vertex.txt", "shader1fragment.txt");
 	ResourceManager->ShaderManager.AddShader("FrameShader", "frameshader_vertex.txt", "frameshader_fragment.txt");
 	Renderer->PushShader("DefaultShader");
 	ResourceManager->TexList.AddTexture(CONST_LOADING_BACKGROUND_BLACK + ".png", CONST_LOADING_BACKGROUND_BLACK);
 	auto texid = ResourceManager->TexList.AddTexture("emoji/Asset 3@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 4@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 5@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 6@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 7@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 8@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 9@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 10@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 11@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 13@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 14@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 15@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 16@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 17@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 18@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 19@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 20@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 21@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 22@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 23@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 24@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 25@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 26@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 27@20x.png");
 	ResourceManager->TexList.AddTexture("emoji/Asset 28@20x.png");
 	ResourceManager->FontManager.AddFont("droid_sans14", "droid_sans14_font_bitmap.png", "droid_sans14_font_charmap.txt");
 	ResourceManager->FontManager.PushFont("droid_sans14");
 	lastFrameTexture = ResourceManager->TexList.AddEmptyTexture("lastFrameTexture", 1280, 720);
 	Renderer->SetOrthoProjection();
 	Renderer->SetFullScreenViewport();
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		faceRenderPair[i].first.SamplerMap[CONST_STRING_TEXTURE_UNIFORM] = "Asset "+boost::lexical_cast<std::string>(i+3)+"@20x.png";
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 0, 0));
 		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 0));
 		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
 		faceRenderPair[i].second.RefreshBuffer();
 	}
 	cap = cv::VideoCapture(0);
 	//cap = cv::VideoCapture("video.mp4");
 	//cap = cv::VideoCapture("bp.mp4");
 	// Check if camera opened successfully
 	if (!cap.isOpened()) {
 		std::cout << "Error opening video stream or file" << std::endl;
 	}
 	cap.set(CV_CAP_PROP_FRAME_WIDTH, 1280);
 	cap.set(CV_CAP_PROP_FRAME_HEIGHT, 720);
 	faceLandmarkDetector.StartTrackProcess();
 	Inited = true;
 }
 void TMyApplication::InnerDeinit()
 {
 	faceLandmarkDetector.StopTrackProcess();
 	cap.release();
    Inited = false;
    if (Console != NULL)
    {
        *Console<<"APP DEINIT\n";
    }
 }
 void TMyApplication::InnerOnTapDown(Vector2f p)
 {
 }
 void TMyApplication::InnerOnTapUp(Vector2f p)
 {
 }
 void TMyApplication::InnerOnTapUpAfterMove(Vector2f p)
 {
 }
 void TMyApplication::InnerOnMove(Vector2f p, Vector2f shift)
 {
 }
 void TMyApplication::OnFling(Vector2f v) 
 {
 }
 void TMyApplication::InnerDraw()
 {
 	glDisable(GL_DEPTH_TEST);
 	//glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
 	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
 	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
 	/*
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		TRenderParamsSetter params(faceRenderPair[i].first);
 		Renderer->DrawTriangleList(faceRenderPair[i].second);
 	}*/
 }
 void TMyApplication::InnerUpdate(size_t dt)
 {
 	cap >> frame; // get a new frame from camera/video or read image
 	if (frame.empty())
 	{
 		return;
 	}
 	cv::Mat image = frame.clone();
 	currentIndex++;
 	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
 #ifdef USE_PREDICTION
 	frameArr.push_back(image);
 	while (frameArr.size() > 40)
 	{
 		lineShift++;
 		frameArr.erase(frameArr.begin());
 	}
 	currentIndex++;
 	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
 	bool newProcessedIndexFound = false;
 	int newProcessedIndex;
 	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
 	{
 		if (faceLandmarkArr[i].valid)
 		{
 			if (!newProcessedIndexFound)
 			{
 				newProcessedIndexFound = true;
 				newProcessedIndex = faceLandmarkArr[i].frameIndex;
 			}
 			else
 			{
 				if (newProcessedIndex < faceLandmarkArr[i].frameIndex)
 				{
 					newProcessedIndex = faceLandmarkArr[i].frameIndex;
 				}
 			}
 		}
 	}
 	if (newProcessedIndexFound && newProcessedIndex > lastProcessedIndex)
 	{
 		indexStep = 0;
 		lastProcessedIndex = newProcessedIndex;
 		prevPassedIndex = currentIndex;
 	}
 	else
 	{
 		if (prevPassedIndex > lastProcessedIndex + indexStep)
 		{
 			indexStep++;
 		}
 	}
 	//std::cout << currentIndex << " " << lastProcessedIndex << " stepped:" << lastProcessedIndex + indexStep << " indexStep: " << indexStep << std::endl;
 	cv::Mat renderImage;
 	if (lastProcessedIndex + indexStep - lineShift < 0)
 	{
 		renderImage = frameArr[0];
 	}
 	else if (lastProcessedIndex + indexStep - lineShift >= frameArr.size())
 	{
 		renderImage = frameArr[frameArr.size() - 1];
 	}
 	else
 	{
 		renderImage = frameArr[lastProcessedIndex + indexStep - lineShift];
 	}
 	renderImage = renderImage.clone();
 #else
 	cv::Mat renderImage = image; 
 #endif
 	cv::Mat cameraMatrix = (cv::Mat_<float>(3, 3) << 9.5434798118121421e+02, 0.0, 640.0, 0.0, 9.5434798118121421e+02, 360.0, 0.0, 0.0, 1.0);
 	cv::Mat distortionCoefficients = (cv::Mat_<float>(5, 1) << 3.5475494652047600e-02, - 5.9818881494543774e-01, 0.0, 0.0, 1.4642154802454528e+00);
 	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
 	{
 		if (faceLandmarkArr[i].valid)
 		{
 			cv::rectangle(renderImage, faceLandmarkArr[i].faceRect, cv::Scalar(0, 0, 255), 1, 4, 0);
 			for (size_t j = 0; j < faceLandmarkArr[i].landmarkArr.size(); j++)
 			{
 				//cv::circle(renderImage, faceLandmarkArr[i].landmarkArr[j], 0.1, cv::Scalar(0, 255, 255), 4, 8, 0);
 			}
 		}
 	}
 	for (size_t index = 0; index < faceLandmarkArr.size(); index++)
 	{
 		if (faceLandmarkArr[index].valid)
 		{
 			faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr, faceLandmarkArr[index].noticeablyChanged, cameraMatrix, distortionCoefficients);
 			for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
 			{
 				cv::ellipse(renderImage, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)), cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
 			}
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[0]), flipVertical(faceStruct[index].boxModelDst[1]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[1]), flipVertical(faceStruct[index].boxModelDst[2]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[2]), flipVertical(faceStruct[index].boxModelDst[3]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[3]), flipVertical(faceStruct[index].boxModelDst[0]), cv::Scalar(0, 255, 0));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[4]), flipVertical(faceStruct[index].boxModelDst[5]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[5]), flipVertical(faceStruct[index].boxModelDst[6]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[6]), flipVertical(faceStruct[index].boxModelDst[7]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[7]), flipVertical(faceStruct[index].boxModelDst[4]), cv::Scalar(0, 255, 0));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[0]), flipVertical(faceStruct[index].boxModelDst[4]), cv::Scalar(0, 255, 0));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[1]), flipVertical(faceStruct[index].boxModelDst[5]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[2]), flipVertical(faceStruct[index].boxModelDst[6]), cv::Scalar(0, 0, 255));
 			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[3]), flipVertical(faceStruct[index].boxModelDst[7]), cv::Scalar(0, 255, 0));
 			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.35f;
 			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.35f;
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 		else
 		{
 			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], { 0,0 }, { 0,0 }, 0);
 			faceRenderPair[index].second.RefreshBuffer();
 		}
 	}
 	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
 	static std::array<char, 1280 * 720 * 3> buffer;
 	int step = renderImage.step;
 	int height = renderImage.rows;
 	int width = renderImage.cols;
 	int channels = 3;
 	char * data = (char *)renderImage.data;
 	for (int i = 0; i < height; i++)
 	{
 		int ci = height - i - 1;
 		memcpy(&buffer[i*width*channels], &(data[ci*step]), width*channels);
 	}
 	glTexImage2D(
 		GL_TEXTURE_2D,
 		0,
 		GL_RGB,
 		1280,
 		720,
 		0,
 		GL_BGR_EXT,
 		GL_UNSIGNED_BYTE,
 		&buffer[0]);
 }
 bool TMyApplication::IsInited()
 {
    return Inited;
 }
@@ -1,75 +1,107 @@
-#ifndef MAIN_CODE_H_INCLUDED
+#ifndef MAIN_CODE_H_INCLUDED
-#define MAIN_CODE_H_INCLUDED
+#define MAIN_CODE_H_INCLUDED
-
+
-#ifdef TARGET_ANDROID
+#ifdef TARGET_ANDROID
-#include "android_api.h"
+#include "android_api.h"
-#endif
+#endif
-
+
-#ifdef TARGET_WIN32
+#ifdef TARGET_WIN32
-#define NOMINMAX 1
+#define NOMINMAX 1
-#include <Windows.h>
+#include <Windows.h>
-#undef NOMINMAX
+#undef NOMINMAX
-#endif
+#endif
-
+
-#include <stdio.h>
+#include <stdio.h>
-#include <stdlib.h>
+#include <stdlib.h>
-#include <math.h>
+#include <math.h>
-
+
-#include "boost/shared_ptr.hpp"
+#include "boost/shared_ptr.hpp"
-#include "boost/thread/thread.hpp"
+#include "boost/thread/thread.hpp"
-#include "boost/assign.hpp"
+#include "boost/assign.hpp"
-#include "boost/bind.hpp"
+#include "boost/bind.hpp"
-
+
-#include "include/Engine.h"
+#include "include/Engine.h"
-
+
-#include "boost/signals2.hpp"//Signal must be included after asio
+#include "boost/signals2.hpp"//Signal must be included after asio
-
+
-using namespace SE;
+#include "../windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h"
-
+#include "../windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h"
-class TMyApplication : public TApplication
+
-{
+
-protected:
+struct FaceStruct
-
+{
-	bool Inited;
+	SE::Vector2f center;
-
+	SE::Vector2f size;
-	float scale = 1.0;
+
-
+	std::array<SE::Vector2f, LANDMARK_POINT_COUNT> preds;
-public:
+
-	TRenderPair lineRenderPair;
+	std::vector<cv::Point3f> modelPoints;
-
+	cv::Mat rotationVector;
-	bool Loaded;
+	cv::Mat translationVector;
-
+	cv::Mat rotationMatrix;
-	TMyApplication() : TApplication(), Loaded(false), Inited(false) { }
+	cv::Mat poseMatrix;
-
+	cv::Mat eulerAngles;
-	virtual void InnerInit();
+
-
+	std::vector<cv::Point3d> boxModelSrc;
-	virtual void InnerDeinit();
+	std::vector<cv::Point2d> boxModelDst;
-
+
-	virtual void InnerDraw();
+	ValueSmoother landmarkSmoothers[2 * LANDMARK_POINT_COUNT];
-
+	ValueSmoother rotationVectorSmoothers[3];
-	virtual void InnerUpdate(size_t dt);
+	ValueSmoother translationVectorSmoothers[3];
-
+
-	bool IsLoaded();
+	FaceStruct();
-
+	FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
-	bool IsInited();
+
-
+	void CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
-	virtual void OnMouseWheel(short int delta);
+	void ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPred, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficientss);
-
+	void InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds);
-	virtual void InnerOnTapDown(Vector2f p);
+};
-
+
-	virtual void InnerOnTapUp(Vector2f p);
+
-
+
-	virtual void InnerOnTapUpAfterMove(Vector2f p);
+using namespace SE;
-
+
-	virtual void InnerOnMove(Vector2f p, Vector2f shift);
+extern const std::string CONST_LOADING_BACKGROUND_BLACK;
-
+
-	virtual void OnFling(Vector2f v);
+class TMyApplication : public TApplication
-
+{
-};
+protected:
-
+
-
+	bool Inited;
-extern TMyApplication* Application;
+
-
+	std::array<TRenderPair, MAX_FACE_COUNT> faceRenderPair;
-
+
-
+public:
-#endif
+
 	TMyApplication() : TApplication(), Inited(false) { }
 	virtual void InnerInit();
 	virtual void InnerDeinit();
 	virtual void InnerDraw();
 	virtual void InnerUpdate(size_t dt);
 	bool IsInited();
 	virtual void InnerOnTapDown(Vector2f p);
 	virtual void InnerOnTapUp(Vector2f p);
 	virtual void InnerOnTapUpAfterMove(Vector2f p);
 	virtual void InnerOnMove(Vector2f p, Vector2f shift);
 	virtual void OnFling(Vector2f v);
 };
 extern TMyApplication* Application;
 #endif
@@ -1,220 +0,0 @@
 #include "menucode.h"
 #include "main_code.h"
 const float CONST_SPEED_NEG_ACCELERATION_K = 0.03f;
 const float CONST_MENU_POS_K = 0.25f;
 const float CONST_MENU_WINDOW_DISTANCE = 240.f + 40.f;
 const float CONST_SHIFT_ON_SIDE = 0.f;
 TGameMenu::TGameMenu()
 	: MenuSpeed(0)
 	, MenuPos(0)
 	, HoldToTap(false)
 	, MenuItemCount(1)
 	, SelectedGame(-1)
 {
 }
 void TGameMenu::Draw()
 {
 //	*SE::Console << "TGameMenu::Draw";
    CheckGlError("Draw TGameMenu");
    RenderUniform1i("sel", 0);
    RenderUniform1f("Transparency", 1.f);
    float bkgShift = MenuPos*0.1f - 100.f;
 	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["main_menu_bkg_left"]);	
 	Renderer->DrawRect(Vector2f(bkgShift,0.f), Vector2f(480.f+bkgShift,320.f));
 	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["main_menu_bkg_right"]);
 	Renderer->DrawRect(Vector2f(480.f+bkgShift,0.f), Vector2f(960.f+bkgShift,320.f));
 	if (SelectedGame == 0)
 	{
        RenderUniform1i("sel", 1);
    }
    else
    {
        RenderUniform1i("sel", 0);
    }
    for (int i=0; i<MenuItemCount; i++)
    {
        glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["levelshot"+tostr(i+1)]);
        Renderer->DrawRect(Vector2f(160.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*i,80.f), Vector2f(160.f+240.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*i, 240.f));
    }
    if (MenuItemCount == 12)
    {
        glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["game_end"]);
        Renderer->DrawRect(Vector2f(160.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*12,160.f - 64.f), Vector2f(160.f+256.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*12, 160.f + 64.f));
    }
    glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["select_level"]);
    Renderer->DrawRect(Vector2f(240.f-128.f, 241.f), Vector2f(240.f+128.f, 305.f));
 	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["logo_small"]);
    Renderer->DrawRect(Vector2f(240.f-128.f+15.f, 0.f), Vector2f(240.f+128.f+15.f, 64.f));
    CheckGlError("Draw TGameMenu 2");
 }
 void TGameMenu::Update(size_t dt)
 {
 	if (HoldToTap)
 		return;
 	// åñëè êíîïêà íå íàæàòà
 	float k;
 	MenuPos = MenuPos + MenuSpeed*dt/1000.f;
    int menuItemToShowCount;
    if (MenuItemCount == 12)
    {
        menuItemToShowCount = 13;
    }
    else
    {
        menuItemToShowCount = MenuItemCount;
    }
 	float acc;
 	if (MenuItemCount == 1)
 	{
 	   k = 3;
    }
 	else
 	{
        k = fabs( - MenuPos - ((menuItemToShowCount-1)*CONST_MENU_WINDOW_DISTANCE*0.5f)) / ((MenuItemCount-1)*CONST_MENU_WINDOW_DISTANCE*0.5f);  
    }
 	acc = -MenuSpeed * CONST_SPEED_NEG_ACCELERATION_K * k;
 	float xleft = (- MenuPos + CONST_SHIFT_ON_SIDE - (menuItemToShowCount-1)*CONST_MENU_WINDOW_DISTANCE);
 	if (xleft < 0)
 	{
        xleft = 0;
    }
    acc += xleft*CONST_MENU_POS_K;
    float xright = (MenuPos - CONST_SHIFT_ON_SIDE);
 	if (xright < 0)
 	{
        xright = 0;
    }
    acc -= xright*CONST_MENU_POS_K;
 	if ((MenuSpeed + acc*dt) * MenuSpeed < 0) //Check if MenuSpeed changes direction
 	{
 		MenuSpeed = 0;
 	}
 	else
 	{
 		MenuSpeed += acc*dt;
 	}
 }
 void TGameMenu::OnTapDown(Vector2f pos)
 {
 	*SE::Console << "TGameMenu::OnTapDown";
    HoldToTap = true;
    if (pos(1)<64.f && pos(0)>=265.f-128.f && pos(0)<=265.f+128.f)
    {
        Application->GoFromMenuToCredits();
        return;
    }
    Vector2f realPos = pos - Vector2f(MenuPos, 0);
    if (realPos(1) >= 80.f && realPos(1) <= 240.f)
    {
        float x = realPos(0) - 160.f;
        int p = 0;
        while (x >280.f)
        {
            x -= 280.f;
            p++;
        }
        if (x <= 240.f && x >= 0.f && (p<MenuItemCount))
        {
            SelectedGame = p;
            return;
        }
    }
    SelectedGame = -1;
 }
 void TGameMenu::OnTapUp(Vector2f pos)
 {
 	*SE::Console << "TGameMenu::OnTapUp";
 	HoldToTap = false;
 	if (SelectedGame != -1)
 	{
        MenuPos = -CONST_MENU_WINDOW_DISTANCE*SelectedGame-40.f;
        Application->GoFromMenuToGame(SelectedGame);
        SelectedGame = -1;
    }
 }
 void TGameMenu::OnTapUpAfterMove(Vector2f pos)
 {
    HoldToTap = false;
 }
 void TGameMenu::OnFling(Vector2f slideSpeed)
 {
 	*SE::Console << "TGameMenu::OnFling";
 	HoldToTap = false;
 	MenuSpeed = slideSpeed(0);
 }
 void TGameMenu::OnScroll(Vector2f shift)
 {
 	MenuPos = MenuPos - shift(0);
 }
 void TGameMenu::OpenNextLevel()
 {
 	*SE::Console << "TGameMenu::OpenNextLevel";
    if (MenuItemCount < 12)
    {
        MenuItemCount++;
    }
 }
 int TGameMenu::GetMenuItemCount()
 {
 	*SE::Console << "TGameMenu::GetMenuItemCount";
    return MenuItemCount;
 }
 void TGameMenu::SetMenuItemCount(int menuItemCount)
 {
 	*SE::Console << "TGameMenu::SetMenuItemCount";
    MenuItemCount = menuItemCount;
 }
@@ -1,41 +0,0 @@
 #ifndef MENUCODE_H_INCLUDED
 #define MENUCODE_H_INCLUDED
 #include "include/Engine.h"
 #include "game_area_interface.h"
 using namespace SE;
 class TGameMenu : public TGameAreaAncestor
 {
 protected:
 	float MenuSpeed;
 	float MenuPos;
 	bool HoldToTap;
 	int MenuItemCount;
 	int SelectedGame;
 public:
 	TGameMenu();
 	virtual void Draw();
 	virtual void Update(size_t dt);
 	virtual void OnTapDown(Vector2f pos);
 	virtual void OnTapUp(Vector2f pos);
    virtual void OnTapUpAfterMove(Vector2f pos);
 	virtual void OnFling(Vector2f slideSpeed);
 	virtual void OnScroll(Vector2f shift);
 	void OpenNextLevel();
 	int GetMenuItemCount();
 	void SetMenuItemCount(int menuItemCount);
 };
 #endif
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <ItemGroup Label="ProjectConfigurations">
    <ProjectConfiguration Include="Debug|Win32">
      <Configuration>Debug</Configuration>
@@ -22,27 +22,28 @@
    <ProjectGuid>{1CC98EEE-BBCB-4D79-B6D7-8511789172C5}</ProjectGuid>
    <Keyword>Win32Proj</Keyword>
    <RootNamespace>DoubleHitBallswin</RootNamespace>
-    <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
    <ProjectName>TorchFaceTracker</ProjectName>
  </PropertyGroup>
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
    <UseDebugLibraries>true</UseDebugLibraries>
    <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
    <UseDebugLibraries>true</UseDebugLibraries>
    <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
    <UseDebugLibraries>false</UseDebugLibraries>
    <WholeProgramOptimization>true</WholeProgramOptimization>
    <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
@@ -111,13 +112,13 @@
      <WarningLevel>Level3</WarningLevel>
      <Optimization>Disabled</Optimization>
      <PreprocessorDefinitions>TARGET_WIN32;DEBUG;_WIN32_WINNT=0x0501;EIGEN_DONT_ALIGN_STATICALLY;_WINDOWS;WIN32;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_63_0/;../../../../boost_1_63_0/bin.v2/libs/system/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/date_time/build/msvc-14.0/debug/address-model-64/link-static/threading-multi/;../../../../boost_1_63_0/bin.v2/libs/regex/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/thread/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/chrono/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/signals/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>../../../../luajit-vlad/luajit-vlad;..\..\..\..\..\Torch7\distro\install\include;C:\Work\TorchProjects\dlib;D:\Work\TorchProjects\dlib\dlib\..;..\..\..\..\..\Torch7\distro\exe\luajit-2.1\src;..\..\..\..\..\opencv3.4\build\include;D:\Work\Torch7\distro\install\include;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
      <DisableSpecificWarnings>4503</DisableSpecificWarnings>
    </ClCompile>
    <Link>
      <SubSystem>Windows</SubSystem>
      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_63_0/stage/x64/lib;../../../../boost_1_63_0/bin.v2/libs/system/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/date_time/build/msvc-14.0/debug/address-model-64/link-static/threading-multi/;../../../../boost_1_63_0/bin.v2/libs/regex/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/thread/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/chrono/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/signals/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Debug</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Debug</AdditionalLibraryDirectories>
      <AdditionalDependencies>libjpeg.lib;libpng.lib;Engine.lib;opengl32.lib;glu32.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
    </Link>
  </ItemDefinitionGroup>
@@ -147,11 +148,11 @@
      <WarningLevel>Level3</WarningLevel>
      <PrecompiledHeader>
      </PrecompiledHeader>
-      <Optimization>MaxSpeed</Optimization>
+      <Optimization>Disabled</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
      <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;TARGET_WIN32;_WIN32_WINNT=0x0501;WIN32_LEAN_AND_MEAN;EIGEN_DONT_ALIGN_STATICALLY;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>../../../../luajit-vlad/luajit-vlad;..\..\..\..\..\Torch7\distro\install\include;C:\Work\TorchProjects\dlib;D:\Work\TorchProjects\dlib\dlib\..;..\..\..\..\..\Torch7\distro\exe\luajit-2.1\src;..\..\..\..\..\opencv3.4\build\include;D:\Work\Torch7\distro\install\include;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
      <DisableSpecificWarnings>4503</DisableSpecificWarnings>
    </ClCompile>
    <Link>
@@ -159,17 +160,21 @@
      <GenerateDebugInformation>true</GenerateDebugInformation>
      <EnableCOMDATFolding>true</EnableCOMDATFolding>
      <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Release</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>..\..\..\..\..\opencv3.4-build\lib\$(Configuration);..\..\..\..\..\opencv3.4-build\3rdparty\lib\$(Configuration);..\..\..\..\..\opencv3.4-build\3rdparty\ippicv\ippicv_win\lib\intel64;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../x64/Release;C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/lib/x64;C:/Program Files/NVIDIA GPU Computing</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libjpeg.lib;libpng.lib;Engine.lib;opengl32.lib;glu32.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>ippiw.lib;ippicvmt.lib;cudart.lib;nppc.lib;nppial.lib;nppicc.lib;nppicom.lib;nppidei.lib;nppif.lib;nppig.lib;nppim.lib;nppist.lib;nppisu.lib;nppitc.lib;npps.lib;cufft.lib;opencv_dnn341.lib;opencv_highgui341.lib;opencv_videoio341.lib;opencv_imgcodecs341.lib;opencv_imgproc341.lib;opencv_core341.lib;opencv_calib3d341.lib;opencv_cudev341.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;comdlg32.lib;advapi32.lib;Engine.lib;opengl32.lib;glu32.lib;libjpeg-static.lib;libpng-static.lib;%(AdditionalDependencies)</AdditionalDependencies>
    </Link>
  </ItemDefinitionGroup>
  <ItemGroup>
    <ClCompile Include="..\..\..\game\main_code.cpp" />
    <ClCompile Include="FaceLandmarkDetector.cpp" />
    <ClCompile Include="main.cpp" />
    <ClCompile Include="ValueSmoother.cpp" />
  </ItemGroup>
  <ItemGroup>
    <ClInclude Include="..\..\..\game\main_code.h" />
    <ClInclude Include="FaceLandmarkDetector.h" />
    <ClInclude Include="main.h" />
    <ClInclude Include="ValueSmoother.h" />
  </ItemGroup>
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
  <ImportGroup Label="ExtensionTargets">
@@ -21,6 +21,12 @@
    <ClCompile Include="..\..\..\game\main_code.cpp">
      <Filter>Файлы исходного кода</Filter>
    </ClCompile>
    <ClCompile Include="FaceLandmarkDetector.cpp">
      <Filter>Файлы исходного кода</Filter>
    </ClCompile>
    <ClCompile Include="ValueSmoother.cpp">
      <Filter>Файлы исходного кода</Filter>
    </ClCompile>
  </ItemGroup>
  <ItemGroup>
    <ClInclude Include="main.h">
@@ -29,5 +35,11 @@
    <ClInclude Include="..\..\..\game\main_code.h">
      <Filter>Заголовочные файлы</Filter>
    </ClInclude>
    <ClInclude Include="FaceLandmarkDetector.h">
      <Filter>Заголовочные файлы</Filter>
    </ClInclude>
    <ClInclude Include="ValueSmoother.h">
      <Filter>Заголовочные файлы</Filter>
    </ClInclude>
  </ItemGroup>
 </Project>
@@ -0,0 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <PropertyGroup />
 </Project>
@@ -0,0 +1,490 @@
 #include "FaceLandmarkDetector.h"
 static const size_t faceTrackerWidth = 300;
 static const size_t faceTrackerHeight = 300;
 static const double faceTrackerScaleFactor = 1.0;
 static const cv::Scalar faceTrackerMeanVal(104.0, 177.0, 123.0);
 float calcSquaredDistanceBetweenFaces(const cv::Rect& face1, const cv::Rect& face2)
 {
 	cv::Point2f oldCenter(face1.x + face1.width*0.5f, face1.y + face1.height*0.5f);
 	cv::Point2f newCenter(face2.x + face2.width*0.5f, face2.y + face2.height*0.5f);
 	float distanceX = newCenter.x - oldCenter.x;
 	float distanceY = newCenter.y - oldCenter.y;
 	float squareDistance = distanceX * distanceX + distanceY * distanceY;
 	return squareDistance;
 }
 double calcSimilarity(const cv::Mat& first, const cv::Mat& second)
 {
 	// assumes that first and second have the same size
 	double errorL2 = cv::norm(first, second, CV_L2);
 	// Convert to a reasonable scale, since L2 error is summed across all pixels of the image.
 	double similarity = errorL2 / static_cast<double>(first.rows * first.cols);
 	return similarity;
 }
 FaceLandmarkDetector::FaceLandmarkDetector()
 {
 	static const cv::String faceTrackerConfiguration = "deploy.prototxt";
 	static const cv::String faceTrackerBinary = "res10_300x300_ssd_iter_140000.caffemodel";
 #ifdef USE_3D_LANDMARKS
 	static const cv::String landmarkTrackerConfiguration = "fa_deploy.prototxt";
 	static const cv::String landmarkTrackerBinary = "fa__iter_1400000.caffemodel";
 #else
 	static const cv::String landmarkTrackerConfiguration = "landmark_deploy.prototxt";
 	static const cv::String landmarkTrackerBinary = "VanFace.caffemodel";
 #endif
 	faceNet = cv::dnn::readNetFromCaffe(faceTrackerConfiguration, faceTrackerBinary);
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		landmarkNetArr[i] = cv::dnn::readNetFromCaffe(landmarkTrackerConfiguration, landmarkTrackerBinary);
 	}
 }
 FaceLandmarkDetector::~FaceLandmarkDetector()
 {
 	StopTrackProcess();
 }
 void FaceLandmarkDetector::StartTrackProcess()
 {
 	if (started)
 	{
 		return;
 	}
 	//Some init setup
 	syncStop = false;
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		syncFaceRectArr[i].valid = false;
 		syncFaceLandmarkArr[i].valid = false;
 	}
 	syncLastFrame = cv::Mat();
 	//Start threads
 	findFaceThread = std::thread([this]() { this->FindFaceThreadProc(); });
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		landmarkThreadArr[i] = std::thread([this, i]() { this->FindLandmarkThreadProc(i); });
 	}
 	started = true;
 }
 void FaceLandmarkDetector::StopTrackProcess()
 {
 	if (!started)
 	{
 		return;
 	}
 	//Do not remove braces
 	{
 		std::unique_lock<std::mutex> lock(syncMutex);
 		syncStop = true;
 		syncFaceRectCond.notify_all();
 	}
 	findFaceThread.join();
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		landmarkThreadArr[i].join();
 	}
 	started = false;
 }
 std::array<FaceLandmarkStruct, MAX_FACE_COUNT> FaceLandmarkDetector::GetFaceLandmarks(cv::Mat frame, int frameIndex)
 {
 	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> result;
 	//Sync area
 	std::unique_lock<std::mutex> lock(syncMutex);
 	// OUT to face rect thread >>>>
 	syncLastFrame = frame;
 	syncLastFrameIndex = frameIndex;
 	// IN from landmark threads <<<<
 	result = syncFaceLandmarkArr;
 	return result;
 }
 void FaceLandmarkDetector::FindFaceThreadProc()
 {
 	cv::Mat localFrame;
 	std::array<FaceIntermediateStruct, MAX_FACE_COUNT> localFaceRectArr;
 	bool localStop = false;
 	int localFrameIndex = 0;
 	while (!syncStop)
 	{
 		if (!localFrame.empty())
 		{
 			std::vector<cv::Rect> faces = InnerFindFaces(localFrame);
 			std::vector<cv::Mat> faceFrames;
 			//Get rid of faces that are too big
 			for (size_t i = 0; i < faces.size(); )
 			{
 				if (faces[i].x < 0 || faces[i].y < 0 || (faces[i].x + faces[i].width >= localFrame.cols) || (faces[i].y + faces[i].height >= localFrame.rows))
 				{
 					faces.erase(faces.begin() + i);
 				}
 				else
 				{
 					i++;
 				}
 			}
 			//Cut faces from frame
 			for (size_t i = 0; i <faces.size(); i++)
 			{
 				faceFrames.push_back(InnerCutFaceFrame(localFrame, faces[i]));
 			}
 			InnerSeparateFacesToBuckets(faces, faceFrames, localFaceRectArr);
 			for (size_t i = 0; i < localFaceRectArr.size(); i++)
 			{
 				if (localFaceRectArr[i].valid)
 				{
 					localFaceRectArr[i].frameIndex = localFrameIndex;
 				}
 			}
 		}
 		//Sync area
 		std::unique_lock<std::mutex> lock(syncMutex);
 		// OUT to landmark Threads >>>>
 		syncFaceRectArr = localFaceRectArr;
 		syncFaceRectCond.notify_all();
 		// IN from main thread <<<<
 		localFrame = syncLastFrame;
 		localFrameIndex = syncLastFrameIndex;
 		localStop = syncStop;
 	}
 }
 void FaceLandmarkDetector::FindLandmarkThreadProc(size_t threadIndex)
 {
 	FaceIntermediateStruct localInputFace;
 	FaceLandmarkStruct localOutputLandmarks;
 	bool localStop = false;
 	while (!localStop)
 	{
 		if (localInputFace.valid)
 		{
 #ifdef USE_3D_LANDMARKS
 			cv::Mat landmarkInputBlob = cv::dnn::blobFromImage(localInputFace.faceFrame, 1.0, cv::Size(40, 40), cv::Scalar(), false, false);
 #else
 			cv::Mat landmarkInputBlob = cv::dnn::blobFromImage(localInputFace.faceFrame, 1.0, cv::Size(60, 60), cv::Scalar(), false, false);
 #endif
 			landmarkInputBlob.reshape(0, std::vector<int>{ 1, 1, localInputFace.faceFrame.rows, localInputFace.faceFrame.cols });
 			landmarkNetArr[threadIndex].setInput(landmarkInputBlob);
 #ifdef USE_3D_LANDMARKS
 			cv::Mat featureBlob = landmarkNetArr[threadIndex].forward("Dense2");
 #else
 			cv::Mat featureBlob = landmarkNetArr[threadIndex].forward("Dense3");
 #endif
 			for (int i = 0; i < featureBlob.cols / 2; i++)
 			{
 				cv::Point landmarkPoint;
 				landmarkPoint.x = int(featureBlob.at<float>(0, 2 * i) * (localInputFace.faceRect.width) + localInputFace.faceRect.x);
 				landmarkPoint.y = int(featureBlob.at<float>(0, 2 * i + 1) * (localInputFace.faceRect.height) + localInputFace.faceRect.y);
 				localOutputLandmarks.landmarkArr[i] = landmarkPoint;
 			}
 			localOutputLandmarks.faceRect = localInputFace.faceRect;
 			localOutputLandmarks.frameIndex = localInputFace.frameIndex;
 			localOutputLandmarks.noticeablyChanged = localInputFace.noticeablyChanged;
 			localOutputLandmarks.valid = true;
 		}
 		else
 		{
 			localOutputLandmarks.valid = false;
 		}
 		//Sync area
 		std::unique_lock<std::mutex> lock(syncMutex);
 		// OUT to main thread >>>>
 		syncFaceLandmarkArr[threadIndex] = localOutputLandmarks;
 		bool doLoop = true;
 		while (doLoop)
 		{
 			//IN from face rect thread <<<<
 			localInputFace = syncFaceRectArr[threadIndex];
 			localStop = syncStop;
 			//Time to quit
 			if (localStop)
 			{
 				doLoop = false;
 			}
 			if (!localInputFace.valid)
 			{
 				syncFaceLandmarkArr[threadIndex].valid = false;
 				syncFaceRectCond.wait(lock);
 			}
 			else
 			{
 				doLoop = false;
 			}
 		}
 	}
 }
 std::vector<cv::Rect> FaceLandmarkDetector::InnerFindFaces(cv::Mat frame)
 {
 	if (frame.channels() == 4)
 	{
 		cvtColor(frame, frame, cv::COLOR_BGRA2BGR);
 	}
 	cv::Mat inputBlob = cv::dnn::blobFromImage(frame, faceTrackerScaleFactor, cv::Size(faceTrackerWidth, faceTrackerHeight), faceTrackerMeanVal, false, false); 
 	faceNet.setInput(inputBlob, "data");
 	cv::Mat detection = faceNet.forward("detection_out");
 	cv::Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
 	float confidenceThreshold = 0.5f;
 	std::vector<cv::Rect> faces;
 	for (int i = 0; i < detectionMat.rows; i++)
 	{
 		float confidence = detectionMat.at<float>(i, 2);
 		if (confidence > confidenceThreshold)
 		{
 			int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frame.cols);
 			int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frame.rows);
 			int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frame.cols);
 			int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frame.rows);
 			cv::Rect object((int)xLeftBottom, (int)yLeftBottom,
 				(int)(xRightTop - xLeftBottom),
 				(int)(yRightTop - yLeftBottom));
 			object.x = object.x - object.width*0.1f;
 			object.y = object.y - object.height*0.1f;
 			object.width = object.width*1.2f;
 			object.height = object.height*1.2f;
 			faces.push_back(object);
 		}
 	}
 	return faces;
 }
 cv::Mat FaceLandmarkDetector::InnerCutFaceFrame(cv::Mat frame, cv::Rect faceRect)
 {
 	cv::Mat srcROI(frame, faceRect);
 	cv::Mat img2;
 	cvtColor(srcROI, img2, CV_RGB2GRAY);
 	img2.convertTo(img2, CV_32FC1);
 #ifdef USE_3D_LANDMARKS
 	cv::Size dsize = cv::Size(40, 40);
 #else
 	cv::Size dsize = cv::Size(60, 60);
 #endif
 	cv::Mat img3 = cv::Mat(dsize, CV_32FC1);
 	cv::resize(img2, img3, dsize, 0, 0, cv::INTER_CUBIC);
 	cv::Mat tmp_m, tmp_sd;
 	double m = 0, sd = 0;
 	meanStdDev(img3, tmp_m, tmp_sd);
 	m = tmp_m.at<double>(0, 0);
 	sd = tmp_sd.at<double>(0, 0);
 	img3 = (img3 - m) / (0.000001 + sd);
 	return img3;
 }
 void FaceLandmarkDetector::InnerSeparateFacesToBuckets(std::vector<cv::Rect>& faceArr, std::vector<cv::Mat>& faceFrameArr, std::array<FaceIntermediateStruct, MAX_FACE_COUNT>& inOutBucketArr)
 {
 	static std::array<std::vector<std::pair<cv::Rect, cv::Mat>>, MAX_FACE_COUNT> facesBelongsToSpace;
 	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
 	{
 		facesBelongsToSpace[i].clear();
 	}
 	//Distribute current faces to nearest old positions
 	for (size_t i = 0; i < faceArr.size(); )
 	{
 		float lastDistance = 1000000.f;
 		int lastIndex = -1;
 		for (size_t j = 0; j < MAX_FACE_COUNT; j++)
 		{
 			if (inOutBucketArr[j].valid)
 			{
 				float newDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, faceArr[i]);
 				if (newDistance < lastDistance)
 				{
 					lastDistance = newDistance;
 					lastIndex = j;
 				}
 			}
 		}
 		if (lastIndex != -1)
 		{
 			facesBelongsToSpace[lastIndex].push_back(std::make_pair(faceArr[i], faceFrameArr[i]));
 			faceArr.erase(faceArr.begin() + i);
 			faceFrameArr.erase(faceFrameArr.begin() + i);
 		}
 		else
 		{
 			i++;
 		}
 	}
 	//Mark "free spots" to those that don't have any face belong to
 	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
 	{
 		if (inOutBucketArr[j].valid)
 		{
 			if (facesBelongsToSpace[j].size() == 0)
 			{
 				inOutBucketArr[j].valid = false;
 			}
 		}
 	}
 	//Find nearest face in each space - others return back to pool
 	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
 	{
 		if (inOutBucketArr[j].valid)
 		{
 			float lastDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, facesBelongsToSpace[j][0].first);
 			int lastIndex = 0;
 			for (size_t i = 1; i < facesBelongsToSpace[j].size(); i++)
 			{
 				float newDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, facesBelongsToSpace[j][i].first);
 				if (newDistance < lastDistance)
 				{
 					lastDistance = newDistance;
 					lastIndex = i;
 				}
 			}
 			inOutBucketArr[j].faceRect = facesBelongsToSpace[j][lastIndex].first;
 			inOutBucketArr[j].faceFrame = facesBelongsToSpace[j][lastIndex].second;
 			inOutBucketArr[j].noticeablyChanged =
 				calcSimilarity(inOutBucketArr[j].faceFrame, facesBelongsToSpace[j][lastIndex].second) > 0.01 ||
 				lastDistance > 100.0;
 			facesBelongsToSpace[j].erase(facesBelongsToSpace[j].begin() + lastIndex);
 			for (size_t k = 0; k < facesBelongsToSpace[j].size(); k++)
 			{
 				faceArr.push_back(facesBelongsToSpace[j][k].first);
 				faceFrameArr.push_back(facesBelongsToSpace[j][k].second);
 			}
 		}
 	}
 	//Distribute faces from pool to empty slots
 	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
 	{
 		if (!inOutBucketArr[j].valid)
 		{
 			if (faceArr.size() > 0)
 			{
 				inOutBucketArr[j].valid = true;
 				inOutBucketArr[j].faceRect = faceArr[0];
 				inOutBucketArr[j].faceFrame = faceFrameArr[0];
 				faceArr.erase(faceArr.begin());
 				faceFrameArr.erase(faceFrameArr.begin());
 			}
 		}
 	}
 }
@@ -0,0 +1,103 @@
 #pragma once
 #include <opencv2/core.hpp>
 #include <opencv2/highgui.hpp>
 #include <opencv2/imgproc.hpp>
 #include <opencv2/calib3d.hpp>
 #include <algorithm>
 #include <memory>
 #include <string>
 #include <vector>
 #include <chrono>
 #include <iostream>
 #include <opencv2/dnn.hpp>
 #include <thread>
 #include <queue>
 #include <mutex>
 #include <condition_variable>
 #define USE_3D_LANDMARKS
 //#include "boost/thread.hpp"
 constexpr size_t LANDMARK_POINT_COUNT = 68;
 constexpr size_t MAX_FACE_COUNT = 5;
 struct FaceIntermediateStruct
 {
 	bool valid = false;
 	cv::Rect faceRect;
 	cv::Mat faceFrame;
 	int frameIndex;
 	bool noticeablyChanged = false;
 };
 struct FaceLandmarkStruct
 {
 	bool valid = false;
 	cv::Rect faceRect;
 	std::array<cv::Point2f, LANDMARK_POINT_COUNT> landmarkArr;
 	int frameIndex;
 	bool noticeablyChanged = false;
 };
 class FaceLandmarkDetector
 {
 public:
 	FaceLandmarkDetector();
 	~FaceLandmarkDetector();
 	void StartTrackProcess();
 	void StopTrackProcess();
 	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> GetFaceLandmarks(cv::Mat frame, int frameIndex);
 protected:
 	cv::dnn::Net faceNet;
 	std::array<cv::dnn::Net, MAX_FACE_COUNT> landmarkNetArr;
 	bool started = false;
 	//Sync variables - shared between threads:
 	cv::Mat syncLastFrame;
 	int syncLastFrameIndex;
 	std::array<FaceIntermediateStruct, MAX_FACE_COUNT> syncFaceRectArr;
 	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> syncFaceLandmarkArr;
 	std::mutex syncMutex;
 	bool syncStop = false;
 	std::condition_variable syncFaceRectCond;
 	std::thread findFaceThread;
 	void FindFaceThreadProc();
 	std::array<std::thread, MAX_FACE_COUNT> landmarkThreadArr;
 	void FindLandmarkThreadProc(size_t threadIndex);
 	std::vector<cv::Rect> InnerFindFaces(cv::Mat frame);
 	cv::Mat InnerCutFaceFrame(cv::Mat frame, cv::Rect faceRect);
 	void InnerSeparateFacesToBuckets(std::vector<cv::Rect>& faceArr, std::vector<cv::Mat>& faceFrameArr, std::array<FaceIntermediateStruct, MAX_FACE_COUNT>& inOutBucketArr);
 };
@@ -0,0 +1,98 @@
 #include "ValueSmoother.h"
 #include <array>
 #include <chrono>
 using namespace std;
 using namespace std::chrono;
 float snapCurve(float x)
 {
 	float y = 1.0 / (x + 1.0);
 	y = (1.0 - y) * 2.0;
 	if (y > 1.0) {
 		return 1.0;
 	}
 	return y;
 }
 void ValueSmoother::reset()
 {
 	inited = false;
 }
 float ValueSmoother::responsiveAnalogReadSimple(float newValue)
 {
 	if (!inited)
 	{
 		inited = true;
 		smoothValue = newValue;
 		return smoothValue;
 	}
 	auto diff = abs(newValue - smoothValue);
 	float snap = snapCurve(diff * SNAP_MULTIPLIER_SIMPLE);
 	smoothValue += (newValue - smoothValue) * snap;
 	return smoothValue;
 }
 void ExtendedValueSmoother::reset()
 {
 	inited = false;
 }
 float ExtendedValueSmoother::responsiveAnalogRead(float newValue)
 {
 	using namespace std::chrono;
 	auto ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
 	auto diff = abs(newValue - smoothValue);
 	// measure the difference between the new value and current value over time
 	// to get a more reasonable indication of how far off the current smooth value is
 	// compared to the actual measurements
 	errorEMA += ((newValue - smoothValue) - errorEMA) * 0.4;
 	// if sleep has been enabled, keep track of when we're asleep or not by marking the time of last activity and testing to see how much time has passed since then
 	if (SLEEP_ENABLE) {
 		// recalculate sleeping status
 		// (asleep if last activity was over SLEEP_DELAY_MS ago)
 		sleeping = abs(errorEMA) < ACTIVITY_THRESHOLD;
 	}
 	if (SLEEP_ENABLE && sleeping) {
 		return smoothValue;
 	}
 	auto snap = snapCurve(diff * SNAP_MULTIPLIER);
 	// when sleep is enabled, the emphasis is stopping on a responsiveValue quickly, and it's less about easing into position. If sleep is enabled, add a small amount to snap so it'll tend to snap into a more accurate position before sleeping starts.
 	if (SLEEP_ENABLE) {
 		snap = snap * 0.5 + 0.5;
 	}
 	// calculate the exponential moving average based on the snap
 	smoothValue += (newValue - smoothValue) * snap;
 	return smoothValue;
 }
@@ -0,0 +1,49 @@
 #pragma once
 class ValueSmoother {
 public:
 	void reset();
 	float responsiveAnalogReadSimple(float newValue);
 protected:
 	bool inited = false;
 	float SNAP_MULTIPLIER_SIMPLE = 1.0 / 60;
 	//float SNAP_MULTIPLIER_SIMPLE = 0.3;
 	float smoothValue = 0;
 };
 struct ExtendedValueSmoother {
 public:
 	void reset();
 	float responsiveAnalogRead(float newValue);
 protected:
 	bool inited = false;
 	float SNAP_MULTIPLIER = 1.0 / 200;
 	float smoothValue = 0;
 	bool SLEEP_ENABLE = true;
 	float ACTIVITY_THRESHOLD = 16.0;
 	float errorEMA = 0;
 	long long lastActivityMS = 0;
 	bool sleeping = false;
 };
@@ -0,0 +1,293 @@
 name: "CNNFacepoint"
 layer {
 	name: "data"
 	type: "Input"
 	top: "data"
 	input_param {
 		shape: {dim: 1 dim: 1 dim: 40 dim: 40}
 	}
 }
 layer {
    name: "Conv1"
    type: "Convolution"
    bottom: "data"
    top: "Conv1"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 16
        pad: 2
        kernel_size: 5
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH1"
  bottom: "Conv1"
  top: "ActivationTangH1"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs1"
  bottom: "ActivationTangH1"
  top: "Abs1"
  type: "AbsVal"
 }
 layer {
    name: "Pool1"
    type: "Pooling"
    bottom: "Abs1"
    top: "Pool1"
    pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
    }
 }
 layer {
    name: "Conv2"
    type: "Convolution"
    bottom: "Pool1"
    top: "Conv2"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 48
        pad: 1
        kernel_size: 3
        stride: 1
        weight_filler {
        type: "xavier"
        std: 0.1
    }
    bias_filler {
        type: "constant"
        value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH2"
  bottom: "Conv2"
  top: "ActivationTangH2"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs2"
  bottom: "ActivationTangH2"
  top: "Abs2"
  type: "AbsVal"
 }
 layer {
    name: "Pool2"
    type: "Pooling"
    bottom: "Abs2"
    top: "Pool2"
    pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
    }
 }
 # layer 3
 layer {
    name: "Conv3"
    type: "Convolution"
    bottom: "Pool2"
    top: "Conv3"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 64
        pad: 0
        kernel_size: 3
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH3"
  bottom: "Conv3"
  top: "ActivationTangH3"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs3"
  bottom: "ActivationTangH3"
  top: "Abs3"
  type: "AbsVal"
 }
 layer {
    name: "Pool3"
    type: "Pooling"
    bottom: "Abs3"
    top: "Pool3"
    pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 2
    }
 }
 # layer 4
 layer {
    name: "Conv4"
    type: "Convolution"
    bottom: "Pool3"
    top: "Conv4"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 64
        pad: 0
        kernel_size: 2
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH4"
  bottom: "Conv4"
  top: "ActivationTangH4"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs4"
  bottom: "ActivationTangH4"
  top: "Abs4"
  type: "AbsVal"
 }
 ########################################
 layer {
    name: "Dense1"
    type: "InnerProduct"
    bottom: "Abs4"
    top: "Dense1"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    inner_product_param {
        num_output: 100
        weight_filler {
            type: "xavier"
        }
        bias_filler {
            type: "constant"
            value: 0
        }
    }
 }
 layer {
  name: "ActivationTangH5"
  bottom: "Dense1"
  top: "ActivationTangH5"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs5"
  bottom: "ActivationTangH5"
  top: "Abs5"
  type: "AbsVal"
 }
 layer {
    name: "Dense2"
    type: "InnerProduct"
    bottom: "Abs5"
    top: "Dense2"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    inner_product_param {
        num_output: 136
        weight_filler {
            type: "xavier"
        }
        bias_filler {
            type: "constant"
            value: 0
        }
    }
 }
@@ -0,0 +1,292 @@
 name: "landmark"
 input: "data"
 input_shape {
  dim: 1
  dim: 1
  dim: 60
  dim: 60
 }
 ########################################
 # the actual net
 # layer 1
 layer {
    name: "Conv1"
    type: "Convolution"
    bottom: "data"
    top: "Conv1"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 20
        pad: 2
        kernel_size: 5
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH1"
  bottom: "Conv1"
  top: "ActivationTangH1"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs1"
  bottom: "ActivationTangH1"
  top: "Abs1"
  type: "AbsVal"
 }
 layer {
    name: "Pool1"
    type: "Pooling"
    bottom: "Abs1"
    top: "Pool1"
    pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
    }
 }
 layer {
    name: "Conv2"
    type: "Convolution"
    bottom: "Pool1"
    top: "Conv2"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 48
        pad: 2
        kernel_size: 5
        stride: 1
        weight_filler {
        type: "xavier"
        std: 0.1
    }
    bias_filler {
        type: "constant"
        value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH2"
  bottom: "Conv2"
  top: "ActivationTangH2"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs2"
  bottom: "ActivationTangH2"
  top: "Abs2"
  type: "AbsVal"
 }
 layer {
    name: "Pool2"
    type: "Pooling"
    bottom: "Abs2"
    top: "Pool2"
    pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
    }
 }
 # layer 3
 layer {
    name: "Conv3"
    type: "Convolution"
    bottom: "Pool2"
    top: "Conv3"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 64
        pad: 0
        kernel_size: 3
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH3"
  bottom: "Conv3"
  top: "ActivationTangH3"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs3"
  bottom: "ActivationTangH3"
  top: "Abs3"
  type: "AbsVal"
 }
 layer {
    name: "Pool3"
    type: "Pooling"
    bottom: "Abs3"
    top: "Pool3"
    pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 2
    }
 }
 # layer 4
 layer {
    name: "Conv4"
    type: "Convolution"
    bottom: "Pool3"
    top: "Conv4"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    convolution_param {
        num_output: 80
        pad: 0
        kernel_size: 3
        stride: 1
        weight_filler {
            type: "xavier"
            std: 0.1
        }
        bias_filler {
            type: "constant"
            value: 0.2
        }
    }
 }
 layer {
  name: "ActivationTangH4"
  bottom: "Conv4"
  top: "ActivationTangH4"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs4"
  bottom: "ActivationTangH4"
  top: "Abs4"
  type: "AbsVal"
 }
 ########################################
 layer {
    name: "Dense1"
    type: "InnerProduct"
    bottom: "Abs4"
    top: "Dense1"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    inner_product_param {
        num_output: 512
        weight_filler {
            type: "xavier"
        }
        bias_filler {
            type: "constant"
            value: 0
        }
    }
 }
 layer {
  name: "ActivationTangH5"
  bottom: "Dense1"
  top: "ActivationTangH5"
  type: "TanH"
 }
 layer {
  name: "ActivationAbs5"
  bottom: "ActivationTangH5"
  top: "Abs5"
  type: "AbsVal"
 }
 layer {
    name: "Dense3"
    type: "InnerProduct"
    bottom: "Abs5"
    top: "Dense3"
    param {
        lr_mult: 1
        decay_mult: 1
    }
    param {
        lr_mult: 2
        decay_mult: 0
    }
    inner_product_param {
        num_output: 136
        weight_filler {
            type: "xavier"
        }
        bias_filler {
            type: "constant"
            value: 0
        }
    }
 }
@@ -4,14 +4,17 @@
 TMyApplication* App = NULL;
 int APIENTRY WinMain(HINSTANCE hCurrentInst, HINSTANCE hPreviousInst,
 	LPSTR lpszCmdLine, int nCmdShow)
 {
-	int width = 512;
+	int width = 1280;
-	int height = 512;
+	int height = 720;
 	if (CreateEngine(width, height))
 	{
@@ -30,5 +33,6 @@ int APIENTRY WinMain(HINSTANCE hCurrentInst, HINSTANCE hPreviousInst,
 		DestroyEngine();
 	}
 	return 0;
 }
Author	SHA1	Message	Date
alexander.biryukov	74402be3ca	Smooth landmarks on significant face position or pose changes	2018-06-04 08:24:42 +05:00
alexander.biryukov	ec0d5090fc	Landmark smoothers	2018-06-01 23:09:53 +05:00
alexander.biryukov	15febc4651	Similarity	2018-06-01 22:31:55 +05:00
alexander.biryukov	601730e0ce	Emphasis head edge	2018-06-01 15:42:10 +05:00
alexander.biryukov	d9bf46043e	Head pose estimation	2018-05-25 21:41:26 +05:00
Vladislav Khorev Dev	7f5aea5047	Fixed building	2017-12-11 18:00:43 +05:00