Smooth landmarks on significant face position or pose changes

.gitignore

@@ -73,6 +73,5 @@ iOSInjectionProject/
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/log.txt
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/progress.txt
 /windows/DoubleHitBalls-win/DoubleHitBalls-win/resources
-/windows/DoubleHitBalls-win/**/*.user
 *.db
 *.opendb

assets/SSAA_4X.frag

@@ -1,19 +0,0 @@
-precision mediump float;
-
-uniform sampler2D Texture;
-uniform vec2 samplesOffset[4];
-
-varying vec2 texCoord;
-
-void main() 
-{
-	vec3 color = vec3(0, 0, 0);
-	for (int i = 0; i < 4; ++i)
-	{
-		color += texture2D(Texture, texCoord + samplesOffset[i]).rgb;
-	}
-	color /= 4;
-
-	//gl_FragColor = vec4(texture2D(Texture, texCoord ), 1.0);
-	gl_FragColor = vec4(color, 1.0);
-}

assets/SSAA_4X.vertex

@@ -1,11 +0,0 @@
-attribute vec3 vPosition;
-attribute vec2 vTexCoord;
-varying vec2 texCoord;
-
-uniform mat4 ProjectionMatrix;
-
-void main() 
-{
-	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
-	texCoord = vTexCoord;
-}

assets/brickshader_fragment.txt

@@ -1,15 +0,0 @@
-precision mediump float;
-uniform sampler2D Texture;
-uniform float Transparency;
-uniform vec4 BrickColor;
-varying vec2 texCoord;
-
-void main() 
-{
-	vec4 color = BrickColor * texture2D(Texture,texCoord).rgba;
-
-	gl_FragColor = vec4(color.rgb, color.a * Transparency);
-	
-	
-	
-}

assets/brickshader_vertex.txt

@@ -1,12 +0,0 @@
-attribute vec3 vPosition;
-attribute vec2 vTexCoord;
-varying vec2 texCoord;
-
-uniform mat4 ProjectionMatrix;
-
-void main() 
-{
-	//480x320
-	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
-	texCoord = vTexCoord;
-}

assets/color_fragment.txt

@@ -1,7 +0,0 @@
-precision mediump float;
-varying vec4 color;
-
-void main() 
-{
-	gl_FragColor = color;
-}

assets/color_vertex.txt

@@ -1,11 +0,0 @@
-attribute vec3 vPosition;
-attribute vec4 vColor;
-varying vec4 color;
-
-uniform mat4 ProjectionMatrix;
-
-void main() 
-{
-	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
-	color = vColor;
-}

assets/emoji/emojis.json

@@ -0,0 +1,249 @@
+{
+	"transform version": "V1",
+	
+	"models": [
+	{
+		"type": "EmojiModel",
+		"name": "Smile"
+	}],
+  
+  "renderParams": [
+    {
+      "vertexShader": "./assets/shaders/vertex/orthoVertexShader.txt",
+      "fragmentShader": "./assets/shaders/fragment/orthoFragmentShader.txt",
+      "alphaTest": "true",
+      "blend": "true",
+      "TAG": "smile & tongue 0"
+    }
+  ],
+	
+	"initialModules": [0, 1, 2, 3, 4],
+
+  "modules": [
+    {
+      "iconPath": "./assets/fx/emoji/Asset 3@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 3@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 24@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 24@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 13@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 13@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 23@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 23@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 19@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 19@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 20@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 20@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 17@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 17@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 18@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 18@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 15@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 15@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 16@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 16@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 21@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 21@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 4@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 4@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 22@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 22@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 14@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 14@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 11@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 11@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 25@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 25@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 26@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 26@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 27@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 27@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 28@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 28@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 5@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 5@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 6@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 6@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 7@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 7@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 8@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 8@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 9@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 9@20x.png"
+        }
+      ]
+    },
+    {
+      "iconPath": "./assets/fx/emoji/Asset 10@20x.png",
+      "suits": [
+        {
+          "renderParamsID": 0,
+          "emojiTexture": "./assets/fx/emoji/Asset 10@20x.png"
+        }
+      ]
+    }
+  ]
+}

assets/ms_fragment.txt

@@ -1,52 +0,0 @@
-precision mediump float;
-varying vec4 color;
-varying vec3 position;
-
-//uniform vec4 lineParams;
-//uniform vec4 startPointEndPoint;
-
-varying vec3 lineParams;
-varying vec4 startPointEndPoint;
-
-
-void main() 
-{
-	float a = lineParams.x;
-	float b = lineParams.y;
-	float c = lineParams.z;
-	
-	float Xa = startPointEndPoint.r;
-	float Ya = startPointEndPoint.g;
-	
-	float Xb = startPointEndPoint.b;
-	float Yb = startPointEndPoint.a;
-	
-	
-	
-	float distance;
-	
-	
-	float distanceToA = (Xa - position.x)*(Xb - Xa) + (Ya - position.y)*(Yb - Ya);
-	
-	float distanceToB = (Xb - position.x)*(Xb - Xa) + (Yb - position.y)*(Yb - Ya);
-	
-	
-	
-	if (distanceToA * distanceToB > 0)
-	{
-		float len = sqrt((Xb - Xa)*(Xb - Xa) + (Yb - Ya)*(Yb - Ya));
-		distance = max(abs(distanceToA/len), abs(distanceToB/len));
-	}
-	else
-	{
-		distance = abs(a*position.x + b*position.y + c);
-	}
-
-	
-	float t = clamp(1.2 / (distance + 1.0) - 0.5, 0.0, 1.0);
-	
-	
-	gl_FragColor = vec4(color.rgb, color.a*t);
-	
-	//gl_FragColor = vec4(color.rgb*clamp(1.2 / (distance + 1.0) - 0.5, 0.0, 1.0), 1.0);
-}

assets/ms_vertex.txt

@@ -1,22 +0,0 @@
-attribute vec3 vPosition;
-attribute vec4 vColor;
-attribute vec3 vLineParams;
-attribute vec4 vStartPointEndPoint;
-
-varying vec3 position;
-varying vec4 color;
-varying vec3 lineParams;
-varying vec4 startPointEndPoint;
-
-
-uniform mat4 ProjectionMatrix;
-
-void main() 
-{
-	gl_Position = ProjectionMatrix * vec4(vPosition.xyz, 1.0);
-	
-	position = vPosition;
-	color = vColor;
-	lineParams = vLineParams;
-	startPointEndPoint = vStartPointEndPoint;
-}

assets/shader1fragment.txt

@@ -7,7 +7,7 @@ void main()
 {
 	vec4 color = texture2D(Texture,texCoord).rgba;
 
-	gl_FragColor = vec4(color.rgb, 1.0);
+	gl_FragColor = vec4(color.rgb, color.a * Transparency);
 	
 	
 	

game/creditscode.cpp

@@ -1,83 +0,0 @@
-#include "creditscode.h"
-#include "include/Engine.h"
-
-#include "main_code.h"
-
-const int TGameCredits::CONST_STATE_APPEARING = 0;
-const int TGameCredits::CONST_STATE_APPEARED = 1;
-const int TGameCredits::CONST_STATE_DISAPPEARING = 2;
-const int TGameCredits::CONST_STATE_DISAPPEARED = 3;
-
-const float TGameCredits::CONST_APPEAR_TIME = 150.f;
-
-TGameCredits::TGameCredits()
-    : TGameAreaAncestor()
-    , StateTimer(0)
-    , State(CONST_STATE_DISAPPEARED)
-{
-}
-
-void TGameCredits::Draw()
-{
-	//*SE::Console << "TGameCredits::Draw";
-    float transparency = StateTimer/CONST_APPEAR_TIME;
-
-    
-    RenderUniform1f("Transparency", transparency);
-    
-	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList["main_menu_bkg_left"]);
-	Renderer->DrawRect(Vector2f(0, 0.f), Vector2f(480.f, 320.f));
-	
-	const Vector2f creditsPos(140.f, 130.f);
-	
-	const Vector2f creditsHalfSize(128.f, 128.f);
-	
-	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_CREDITS_TEXTURE]);
-	Renderer->DrawRect(creditsPos-creditsHalfSize, creditsPos+creditsHalfSize);
-	
-	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOGO_SMALL_TEXTURE]);
-	Renderer->DrawRect(Vector2f(240.f-128.f+15.f, 320.f-64.f), Vector2f(240.f+128.f+15.f, 320.f));
-    
-    CheckGlError("Draw TGameCredits");
-}
-
-void TGameCredits::Update(size_t dt)
-{
-    if (State == CONST_STATE_APPEARING)
-	{
-		*SE::Console << "TGameCredits::Update - CONST_STATE_APPEARING";
-        StateTimer += dt;
-        if (StateTimer >= CONST_APPEAR_TIME)
-        {
-            StateTimer = CONST_APPEAR_TIME;
-            State = CONST_STATE_APPEARED;
-        }
-    }
-    else if (State == CONST_STATE_DISAPPEARING)
-	{
-        StateTimer -= dt;
-        if (StateTimer <= 0.f)
-        {
-            StateTimer = 0.f;
-            State = CONST_STATE_DISAPPEARED;
-        }
-    }
-}
-
-void TGameCredits::OnTapDown(Vector2f p)
-{
-	*SE::Console << "TGameCredits::OnTapDown";
-    Application->GoFromCreditsToMenu();
-}
-
-void TGameCredits::StartAppear()
-{
-	*SE::Console << "TGameCredits::StartAppear";
-    State = CONST_STATE_APPEARING;
-}
-
-void TGameCredits::StartDisappear()
-{
-	*SE::Console << "TGameCredits::StartDisappear";
-    State = CONST_STATE_DISAPPEARING;
-}

game/creditscode.h

@@ -1,34 +0,0 @@
-#ifndef CREDITSCODE_H_INCLUDED
-#define CREDITSCODE_H_INCLUDED
-
-#include "game_area_interface.h"
-
-using namespace SE;
-
-
-class TGameCredits : public TGameAreaAncestor
-{
-    static const int CONST_STATE_APPEARING;
-    static const int CONST_STATE_APPEARED;
-    static const int CONST_STATE_DISAPPEARING;
-    static const int CONST_STATE_DISAPPEARED;
-    static const float CONST_APPEAR_TIME;
-
-
-    int State;
-    float StateTimer;
-
-public:
-    
-    TGameCredits();
-    
-    virtual void Draw();
-    virtual void Update(size_t dt);
-    virtual void OnTapDown(Vector2f p);
-    
-    void StartAppear();
-    void StartDisappear();
-};
-
-
-#endif

game/game_area_interface.h

@@ -1,25 +0,0 @@
-#ifndef GAME_AREA_INTERFACE_H_INCLUDED
-#define GAME_AREA_INTERFACE_H_INCLUDED
-
-#include "include/Engine.h"
-
-using namespace SE;
-
-
-class TGameAreaAncestor
-{
-public:
-    virtual void Draw() { }
-    virtual void Update(size_t) { }
-    virtual void OnTapUp(Vector2f) { }
-    virtual void OnTapUpAfterMove(Vector2f) { }
-    
-    virtual void OnTapDown(Vector2f) { }
-    virtual void OnFling(Vector2f) { }
-	virtual void OnScroll(Vector2f) { }
-	
-	virtual ~TGameAreaAncestor() { } //By default destructor is not virtual
-};
-
-
-#endif

game/gamecode.h

@@ -1,205 +0,0 @@
-#ifndef GAMECODE_H_INCLUDED
-#define GAMECODE_H_INCLUDED
-
-#include "include/Engine.h"
-#include "game_area_interface.h"
-
-#include <list>
-
-using namespace SE;
-
-
-const int CONST_BRICKSTATE_VISIBLE = 3;
-const int CONST_BRICKSTATE_DISAPPEAR = 2;
-const int CONST_BRICKSTATE_INVISIBLE = 1;
-const int CONST_BRICKSTATE_APPEAR = 0;
-
-const float CONST_BRICK_DISAPPEAR_TIME = 200.f;
-const float CONST_BRICK_APPEAR_TIME = 200.f;
-
-const int CONST_BRICKMATRIX_WIDTH = 12;
-const int CONST_BRICKMATRIX_HEIGHT = 13;
-const float CONST_BRICK_WIDTH = 37.5f;
-const float CONST_BRICK_HEIGHT = 0.5f*CONST_BRICK_WIDTH;
-const float CONST_BRICK_SHIFT_X = 15.f;
-const float CONST_BRICK_SHIFT_Y = -16.f;
-
-const float CONST_BONUS_APPEAR_TIME = 400.f;
-const float CONST_BONUS_FALL_SPEED = 100.f;
-const float CONST_BONUS_GOTHROUGH_TIME = 8000.f;
-const float CONST_BONUS_CATCH_DISTANCE_X = 65.f;
-const float CONST_BONUS_CATCH_DISTANCE_Y = 20.f;
-
-const float CONST_BONUS_FLOOR_TIME = 8000.f;
-const float CONST_BONUS_FLOOR_APPEAR_SPEED = 50.f;
-
-const int CONST_BONUS_TYPE_MULTIPLIER = 0;
-const int CONST_BONUS_TYPE_GOTHROUGH = 1;
-const int CONST_BONUS_TYPE_FLOOR = 2;
-
-
-class TBrick
-{
-protected:
-    int InitialLocked;
-    int Locked; //0, 1 or 2
-    int State;
-    float StateTimer;
-    Vector4f Color;
-public:
-    TBrick();
-    void SetVisible(Vector4f color, int locked);
-    void SetInvisible();
-    void TryDrawAppear(int ipos, int jpos);
-    void Update(size_t dt);
-    Vector2f GetPosFrom(int ipos, int jpos);
-    Vector4f GetColor();
-    void Appear(Vector4f color, int locked);
-    void Appear();
-    void Hit();
-    void Disappear();
-    int GetLocked();
-    bool CanReact();
-};
-
-class TBonusFalling
-{
-protected:
-   float Lifetime;
-   Vector2f Pos;
-   std::string TexName;
-   int BonusType; //0 - multiplier,  1 - Go-through, 2 - floor
-public:
-    TBonusFalling(Vector2f pos);
-    Vector2f GetPos();
-    int GetType();
-    void Draw();
-    void Update(size_t dt);           
-};
-
-struct TBall
-{
-    Vector2f Pos;
-    Vector2f Velocity;
-    Vector4f Color;
-    std::list<Vector2f> TalePos;
-    
-    Vector2f BallInBlock;
-    Vector2f PrevBallInBlock;
-    
-    TBall(Vector2f pos, Vector2f velocity, Vector4f color);
-    Vector2f GetPos();
-    Vector2f GetVelocityVector();
-    
-    void Go();
-    
-    void ReflectToLeft();
-    void ReflectToRight();
-    void ReflectToUp();
-    void ReflectToDown();
-    
-    void TryReflectOnReflector(Vector2f refPos);
-   
-    void Update(size_t dt);    
-};
-
-typedef std::pair<Vector4f, std::string> PairColorTexture;
-
-struct TBlockInstansingList
-{
-    //color-texture  ---> and list of triangles for this color
-    std::list<std::pair<PairColorTexture, TTriangleList>> ColorBlockList;
-};
-
-
-struct TBallInstancingList
-{
-    std::map<int, TTriangleList> BallAndGlowList;
-};
-
-class TGameLevel : public TGameAreaAncestor
-{
-protected:
-    std::string BkgTexture;
-    std::string LevelScreenTexture;
-    std::string LevelFileName;
-    
-    Vector2f ReflectorPos;
-    
-    int LevelState;
-    bool PrevLevelStateIsStandby;
-    float StateTimer;
-    
-    TBrick BlockMatrix[CONST_BRICKMATRIX_WIDTH][CONST_BRICKMATRIX_HEIGHT];
-    TBlockInstansingList BlockInstansingList;
-    
-    bool TapInBackBtnArea(const Vector2f& pos);
-    
-    void ReloadBlockInstansingList();
-    void SetFinished();
-    void SetFinishFreeze();
-    
-    void DrawBuffer();
-    void DrawPauseButtons();
-    
-    void DrawBallInstancingList();
- 
-    bool RenderBufferReady;
-    
-    float OutScale;
-    
-    float OutScaleVelocity;
-    
-    void TryGoToMenu();
-    
-    std::list<TBonusFalling> BonusFallingList;
-    
-    std::list<TBall> BallList;
-    
-    TBallInstancingList BallInstancingList;
-    
-    Vector4f BallColor;
-    
-    void ReloadBallInstancingList();
-    void RefreshBallInstancingList();
-    
-    
-    void UpdateBallList(size_t dt);
-    void MultiplyBalls(Vector2f pos, Vector2f velocity);
-    
-    Vector2f GetBlock(const Vector2f& pos);
-    void InitLevel();
-    
-    float BonusGothroughTimer;
-    float BonusFloorTimer;
-    float BonusFloorPosY;
-    
-    Vector4f ParseColor(const std::string& s);
-    void ReloadLevel();
-    
-public:
-    TGameLevel();
-    ~TGameLevel();
-    
-    void FillWithFile(const std::string& filename);
-    
-    void SetStandBy();
-    void SetLoading(const std::string& bkg, const std::string& levelscreen);
-    bool IsLoaded();
-    
-	virtual void Draw();
-	
-	void SetPause();
-	bool IsPaused();
-	void ReleasePause();
-	
-	virtual void Update(size_t dt);
-	virtual void OnTapDown(Vector2f pos);
-	virtual void OnTapUp(Vector2f pos);
-	virtual void OnFling(Vector2f slideSpeed);
-	virtual void OnScroll(Vector2f shift);
-};
-
-
-
-#endif

game/loadingcode.cpp

@@ -1,20 +0,0 @@
-#include "loadingcode.h"
-#include "include/Engine.h"
-#include "main_code.h"
-
-void TGameLoading::Draw()
-{
-	//*SE::Console << "TGameLoading::Draw";
-	
-	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
-	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
-
-    glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOADING_TEXTURE]);
-	Renderer->DrawRect(Vector2f(240.f - 128.f, 10.f), Vector2f(240.f + 128.f, 10.f + 64.f));
-	
-	glBindTexture(GL_TEXTURE_2D,ResourceManager->TexList[CONST_LOGO_SMALL_TEXTURE]);
-	Renderer->DrawRect(Vector2f(480.f - 256.f+50.f, 320.f - 64.f), Vector2f(480.f+50.f, 320.f));
-    
-    CheckGlError();
-
-}

game/loadingcode.h

@@ -1,15 +0,0 @@
-#ifndef LOADINGCODE_H_INCLUDED
-#define LOADINGCODE_H_INCLUDED
-
-#include "game_area_interface.h"
-
-using namespace SE;
-
-class TGameLoading : public TGameAreaAncestor
-{
-public:
-    virtual void Draw();
-};
-
-
-#endif

game/main_code — копия.cpp

@@ -0,0 +1,592 @@
+#include "main_code.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <time.h>
+
+#include "include/Engine.h"
+
+#include "main_code.h"
+
+
+
+
+FaceStruct::FaceStruct()
+{
+}
+
+FaceStruct::FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
+{
+	ApplyPreds(fromPreds);
+	
+}
+
+void FaceStruct::CalcFromPreds()
+{
+	float minX = preds[0](0);
+	float maxX = preds[0](0);
+	float minY = preds[0](1);
+	float maxY = preds[0](1);
+	float sumX = preds[0](0);
+	float sumY = preds[0](1);
+
+	for (size_t i = 1; i < LANDMARK_POINT_COUNT; i++)
+	{
+		if (minX > preds[i](0))
+		{
+			minX = preds[i](0);
+		}
+
+		if (minY > preds[i](1))
+		{
+			minY = preds[i](1);
+		}
+
+		if (maxX < preds[i](0))
+		{
+			maxX = preds[i](0);
+		}
+
+		if (maxY < preds[i](1))
+		{
+			maxY = preds[i](1);
+		}
+
+		sumX += preds[i](0);
+		sumY += preds[i](1);
+	}
+
+	float avgX = sumX / static_cast<float>(LANDMARK_POINT_COUNT);
+	float avgY = sumY / static_cast<float>(LANDMARK_POINT_COUNT);
+
+
+
+	center = { avgX, avgY };
+	size = { maxX- minX,maxY - minY };
+
+	//center = { centerXSmoother.responsiveAnalogReadSimple(avgX), centerYSmoother.responsiveAnalogReadSimple(avgY) };
+	//size = { sizeWidthSmoother.responsiveAnalogReadSimple(maxX- minX), sizeHeightSmoother.responsiveAnalogReadSimple(maxY - minY) };
+}
+
+void FaceStruct::ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
+{
+	InnerApplyPreds(fromPreds);
+	CalcFromPreds();
+
+	/*
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+	{
+		preds[i] = preds[i] - center;
+
+	}
+
+
+	historicalPreds.push_back(preds);
+	if (historicalPreds.size() > 4)
+	{
+		historicalPreds.erase(historicalPreds.begin());
+	}
+
+
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+	{
+		preds[i] = { 0.f, 0.f };
+		for (size_t x = 0; x < historicalPreds.size(); x++)
+		{
+			preds[i] += historicalPreds[x][i];
+		}
+
+		preds[i] = preds[i] / static_cast<float>(historicalPreds.size());
+	}
+
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+	{
+		preds[i] = preds[i] + center;
+	}*/
+}
+
+void FaceStruct::InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
+{
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+	{
+		preds[i] = { fromPreds[i].x, 720 - fromPreds[i].y};
+		
+	}	
+}
+
+
+
+
+cv::VideoCapture cap;
+
+FaceLandmarkDetector faceLandmarkDetector;
+
+GLuint lastFrameTexture;
+
+
+std::array<FaceLandmarkStruct, MAX_FACE_COUNT> faceLandmarkArr;
+
+cv::Mat frame;
+
+std::vector<cv::Mat> frameArr;
+
+int currentIndex = -1;
+int lastProcessedIndex = 0;
+int lineShift = 0;
+
+int prevPassedIndex = 0;
+
+int indexStep = 0;
+
+std::array<FaceStruct, MAX_FACE_COUNT> faceStruct;
+
+
+const std::string CONST_LOADING_BACKGROUND_BLACK = "loading_background_black";
+
+
+TMyApplication* Application;
+
+
+void TMyApplication::InnerInit()
+{
+
+    Application = this;
+    
+#ifdef TARGET_WIN32
+#ifdef NDEBUG
+	//ST::PathToResources = "resources/";
+	ST::PathToResources = "../../../assets/";
+#else
+	ST::PathToResources = "../../../assets/";
+#endif
+#endif
+    
+#ifdef TARGET_IOS
+    ST::PathToResources = "assets/";
+#endif
+
+    if (Console != NULL)
+    {
+        *Console<<"APP INIT\n";
+    }
+    srand (static_cast<size_t>(time(NULL)));
+
+	ResourceManager->ShaderManager.AddShader("DefaultShader", "shader1vertex.txt", "shader1fragment.txt");
+	ResourceManager->ShaderManager.AddShader("FrameShader", "frameshader_vertex.txt", "frameshader_fragment.txt");
+	Renderer->PushShader("DefaultShader");
+
+	ResourceManager->TexList.AddTexture(CONST_LOADING_BACKGROUND_BLACK + ".png", CONST_LOADING_BACKGROUND_BLACK);
+
+
+	auto texid = ResourceManager->TexList.AddTexture("emoji/Asset 3@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 4@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 5@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 6@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 7@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 8@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 9@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 10@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 11@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 13@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 14@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 15@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 16@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 17@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 18@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 19@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 20@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 21@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 22@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 23@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 24@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 25@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 26@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 27@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 28@20x.png");
+
+
+	ResourceManager->FontManager.AddFont("droid_sans14", "droid_sans14_font_bitmap.png", "droid_sans14_font_charmap.txt");
+	ResourceManager->FontManager.PushFont("droid_sans14");
+
+
+	lastFrameTexture = ResourceManager->TexList.AddEmptyTexture("lastFrameTexture", 1280, 720);
+
+	Renderer->SetOrthoProjection();
+
+	Renderer->SetFullScreenViewport();
+
+
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		faceRenderPair[i].first.SamplerMap[CONST_STRING_TEXTURE_UNIFORM] = "Asset "+boost::lexical_cast<std::string>(i+3)+"@20x.png";
+
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 0, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
+
+
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 0));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
+		faceRenderPair[i].second.RefreshBuffer();
+	}
+
+
+	cap = cv::VideoCapture(0);
+	//cap = cv::VideoCapture("video.mp4");
+	//cap = cv::VideoCapture("bp.mp4");
+
+
+	// Check if camera opened successfully
+	if (!cap.isOpened()) {
+		std::cout << "Error opening video stream or file" << std::endl;
+	}
+
+	cap.set(CV_CAP_PROP_FRAME_WIDTH, 1280);
+	cap.set(CV_CAP_PROP_FRAME_HEIGHT, 720);
+
+
+		
+	faceLandmarkDetector.StartTrackProcess();
+
+	Inited = true;
+
+
+
+	
+}
+
+void TMyApplication::InnerDeinit()
+{
+
+	faceLandmarkDetector.StopTrackProcess();
+	cap.release();
+
+    Inited = false;
+    
+    if (Console != NULL)
+    {
+        *Console<<"APP DEINIT\n";
+    }
+
+
+}
+
+void TMyApplication::InnerOnTapDown(Vector2f p)
+{
+
+}
+
+void TMyApplication::InnerOnTapUp(Vector2f p)
+{
+
+}
+
+void TMyApplication::InnerOnTapUpAfterMove(Vector2f p)
+{
+
+}
+
+void TMyApplication::InnerOnMove(Vector2f p, Vector2f shift)
+{
+
+}
+	
+void TMyApplication::OnFling(Vector2f v) 
+{
+}
+
+
+	
+void TMyApplication::InnerDraw()
+{
+	glDisable(GL_DEPTH_TEST);
+
+
+
+	//glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
+	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
+	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
+
+
+	/*
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		TRenderParamsSetter params(faceRenderPair[i].first);
+		Renderer->DrawTriangleList(faceRenderPair[i].second);
+	}*/
+	
+
+
+
+}
+
+
+void TMyApplication::InnerUpdate(size_t dt)
+{
+	
+	
+	/*cap >> lastFrame;
+
+	if (lastFrame.empty())
+	{
+		return;
+	}
+
+	
+	faceLandmarkTracker.HandleOneFrame(lastFrame, localInputStructArr, localFaceShift, localOutputStructArr);
+
+
+
+	high_resolution_clock::time_point t2 = high_resolution_clock::now();
+
+	auto duration = duration_cast<microseconds>(t2 - FaceLandmarkTracker::lastTimePoint).count();
+
+	FaceLandmarkTracker::lastTimePoint = t2;
+
+	std::stringstream ofs;
+
+	ofs << round(1000000.0 / duration);
+
+
+	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
+	{
+		if (localOutputStructArr[index].valid && localInputStructArr[index].valid)
+		{
+			cv::Rect oldFaceRect = localOutputStructArr[index].face;
+			cv::Rect newFaceRect = localInputStructArr[index].face;
+			cv::Point2f oldCenter(oldFaceRect.x + oldFaceRect.width*0.5f, oldFaceRect.y + oldFaceRect.height*0.5f);
+			cv::Point2f newCenter(newFaceRect.x + newFaceRect.width*0.5f, newFaceRect.y + newFaceRect.height*0.5f);
+
+			localFaceShift[index] = newCenter - oldCenter;
+		}
+		else
+		{
+			localFaceShift[index] = cv::Point2f(0, 0);
+		}
+	}
+
+	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
+	{
+		if (localOutputStructArr[index].valid)
+		{
+
+			
+			faceStruct[index].ApplyPreds(localOutputStructArr[index].preds);
+
+
+			for (size_t i = 0; i < CONST_PREDS_COUNT; i++)
+			{
+				//cv::ellipse(lastFrame, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)) + localFaceShift[index], cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
+			}
+
+
+			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.25f;
+			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.25f;
+
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+		else
+		{
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], {0,0}, { 0,0 }, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+
+	}
+
+	
+
+	for (size_t index = 0; index < CONST_FACE_COUNT; index++)
+	{
+		if (localInputStructArr[index].valid)
+		{
+			//cv::rectangle(lastFrame, localInputStructArr[index].face, cv::Scalar(255, 0, 255), 4, 8, 0);
+		}
+	}
+	
+	*/
+
+
+
+	cap >> frame; // get a new frame from camera/video or read image
+
+	if (frame.empty())
+	{
+		return;
+	}
+
+	cv::Mat image = frame.clone();
+
+	frameArr.push_back(image);
+
+	while (frameArr.size() > 40)
+	{
+		lineShift++;
+		frameArr.erase(frameArr.begin());
+	}
+
+	currentIndex++;
+
+	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
+
+
+	bool newProcessedIndexFound = false;
+	int newProcessedIndex;
+
+	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
+	{
+		if (faceLandmarkArr[i].valid)
+		{
+			if (!newProcessedIndexFound)
+			{
+				newProcessedIndexFound = true;
+				newProcessedIndex = faceLandmarkArr[i].frameIndex;
+			}
+			else
+			{
+				if (newProcessedIndex < faceLandmarkArr[i].frameIndex)
+				{
+					newProcessedIndex = faceLandmarkArr[i].frameIndex;
+				}
+			}
+		}
+	}
+
+
+	if (newProcessedIndexFound && newProcessedIndex > lastProcessedIndex)
+	{
+		indexStep = 0;
+
+		lastProcessedIndex = newProcessedIndex;
+		prevPassedIndex = currentIndex;
+
+	}
+	else
+	{
+		if (prevPassedIndex > lastProcessedIndex + indexStep)
+		{
+			indexStep++;
+		}
+	}
+
+	//std::cout << currentIndex << " " << lastProcessedIndex << " stepped:" << lastProcessedIndex + indexStep << " indexStep: " << indexStep << std::endl;
+
+
+	cv::Mat renderImage;
+
+	if (lastProcessedIndex + indexStep - lineShift < 0)
+	{
+		renderImage = frameArr[0];
+	}
+	else if (lastProcessedIndex + indexStep - lineShift >= frameArr.size())
+	{
+		renderImage = frameArr[frameArr.size() - 1];
+	}
+	else
+	{
+		renderImage = frameArr[lastProcessedIndex + indexStep - lineShift];
+	}
+
+	renderImage = renderImage.clone();
+
+
+
+
+	
+	cv::Rect frameRect(cv::Point(), renderImage.size());
+
+	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
+	{
+		if (faceLandmarkArr[i].valid)
+		{
+			cv::rectangle(renderImage, faceLandmarkArr[i].faceRect, cv::Scalar(0, 0, 255), 1, 4, 0);
+
+			for (size_t j = 0; j < faceLandmarkArr[i].landmarkArr.size(); j++)
+			{
+
+				//cv::circle(renderImage, faceLandmarkArr[i].landmarkArr[j], 0.1, cv::Scalar(0, 255, 255), 4, 8, 0);
+
+			}
+
+		}
+	}
+
+	for (size_t index = 0; index < faceLandmarkArr.size(); index++)
+	{
+		if (faceLandmarkArr[index].valid)
+		{
+
+
+			faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr);
+
+			
+			for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+			{
+				cv::ellipse(renderImage, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)), cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
+			}
+
+
+			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.35f;
+			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.35f;
+
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+		else
+		{
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], { 0,0 }, { 0,0 }, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+
+	}
+
+
+	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
+
+	static std::array<char, 1280 * 720 * 3> buffer;
+
+	int step = renderImage.step;
+	int height = renderImage.rows;
+	int width = renderImage.cols;
+	int channels = 3;
+	char * data = (char *)renderImage.data;
+	for (int i = 0; i < height; i++)
+	{
+		int ci = height - i - 1;
+		memcpy(&buffer[i*width*channels], &(data[ci*step]), width*channels);
+	}
+
+
+	glTexImage2D(
+		GL_TEXTURE_2D,
+		0,
+		GL_RGB,
+		1280,
+		720,
+		0,
+		GL_BGR_EXT,
+		GL_UNSIGNED_BYTE,
+		&buffer[0]);
+}
+
+
+bool TMyApplication::IsInited()
+{
+    return Inited;
+}

game/main_code.cpp

@@ -9,74 +9,185 @@
 
 #include "main_code.h"
 
-TMyApplication* Application;
+//#define USE_PREDICTION
 
-static const float BUFFER_DISTANCE = 50.f;
+cv::Point flipVertical(cv::Point point) {
-
+	return cv::Point(point.x, 720 - point.y);
-
-TRenderPair linesToRenderBoxes(const std::vector<std::pair<Vector2f, Vector2f>>& lineArr)
-{
-	TRenderPair pair;
-
-	for (size_t i = 0; i < lineArr.size(); i++)
-	{
-
-		Vector2f startPoint = lineArr[i].first;
-		Vector2f endPoint = lineArr[i].second;
-
-		Vector2f renderAreaStart{ std::min<float>(startPoint(0), endPoint(0)) - BUFFER_DISTANCE, std::min<float>(startPoint(1), endPoint(1)) - BUFFER_DISTANCE };
-
-		Vector2f renderAreEnd{ std::max<float>(startPoint(0), endPoint(0)) + BUFFER_DISTANCE, std::max<float>(startPoint(1), endPoint(1)) + BUFFER_DISTANCE };
-
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreaStart(1), 0));
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreEnd(1), 0));
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreEnd(1), 0));
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreaStart(0), renderAreaStart(1), 0));
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreEnd(1), 0));
-		pair.second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(renderAreEnd(0), renderAreaStart(1), 0));
-
-
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-		pair.second.Data.Vec4CoordArr[CONST_STRING_COLOR_ATTRIB].push_back(Vector4f(1, 0, 0, 1));
-
-		Vector2f lineVec = endPoint - startPoint;
-		Vector2f lineNormVec = { -lineVec(1), lineVec(0) };
-
-		lineNormVec.normalize();
-
-		//float distance = abs(normVec.x*(position.x - startPoint.x) + normVec.y*(position.y - startPoint.y));
-		//distance = a*x + b*y + c, here we calculate a, b,c and pass them to shader
-		float a = lineNormVec(0);
-		float b = lineNormVec(1);
-		float c = -(lineNormVec(0)*startPoint(0) + lineNormVec(1)*startPoint(1));
-
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		pair.second.Data.Vec3CoordArr["vLineParams"].push_back(Vector3f(a, b, c));
-		
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		pair.second.Data.Vec4CoordArr["vStartPointEndPoint"].push_back(Vector4f(startPoint(0), startPoint(1), endPoint(0), endPoint(1)));
-		
-		
-	}
-
-	pair.second.RefreshBuffer();
-	pair.first = TRenderParams().withShader("AntialiasingShader");
-
-	return pair;
 }
 
+FaceStruct::FaceStruct()
+{
+	modelPoints.push_back(cv::Point3d(6.825897, 6.760612, 4.402142));     // left brow left corner
+	modelPoints.push_back(cv::Point3d(1.330353, 7.122144, 6.903745));     // left brow right corner
+	modelPoints.push_back(cv::Point3d(-1.330353, 7.122144, 6.903745));    // right brow left corner
+	modelPoints.push_back(cv::Point3d(-6.825897, 6.760612, 4.402142));    // right brow right corner
+	modelPoints.push_back(cv::Point3d(5.311432, 5.485328, 3.987654));     // left eye left corner
+	modelPoints.push_back(cv::Point3d(1.789930, 5.393625, 4.413414));     // left eye right corner
+	modelPoints.push_back(cv::Point3d(-1.789930, 5.393625, 4.413414));    // right eye left corner
+	modelPoints.push_back(cv::Point3d(-5.311432, 5.485328, 3.987654));    // right eye right corner
+	modelPoints.push_back(cv::Point3d(2.005628, 1.409845, 6.165652));     // nose left corner
+	modelPoints.push_back(cv::Point3d(-2.005628, 1.409845, 6.165652));    // nose right corner
+	modelPoints.push_back(cv::Point3d(2.774015, -2.080775, 5.048531));    // mouth left corner
+	modelPoints.push_back(cv::Point3d(-2.774015, -2.080775, 5.048531));   // mouth right corner
+	modelPoints.push_back(cv::Point3d(0.000000, -3.116408, 6.097667));    // mouth central bottom corner
+	modelPoints.push_back(cv::Point3d(0.000000, -7.415691, 4.070434));    // chin corner
+
+	poseMatrix = cv::Mat(3, 4, CV_64FC1);
+	eulerAngles = cv::Mat(3, 1, CV_64FC1);
+
+	boxModelSrc.push_back(cv::Point3d(10.0, 10.0, 10.0));
+	boxModelSrc.push_back(cv::Point3d(10.0, 10.0, -10.0));
+	boxModelSrc.push_back(cv::Point3d(10.0, -10.0, -10.0));
+	boxModelSrc.push_back(cv::Point3d(10.0, -10.0, 10.0));
+	boxModelSrc.push_back(cv::Point3d(-10.0, 10.0, 10.0));
+	boxModelSrc.push_back(cv::Point3d(-10.0, 10.0, -10.0));
+	boxModelSrc.push_back(cv::Point3d(-10.0, -10.0, -10.0));
+	boxModelSrc.push_back(cv::Point3d(-10.0, -10.0, 10.0));
+}
+
+FaceStruct::FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
+{
+	FaceStruct();
+	ApplyPreds(fromPreds, 0.0, cameraMatrix, distortionCoefficients);
+}
+
+void FaceStruct::CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
+{
+	for(size_t i = 0; i < LANDMARK_POINT_COUNT; i++) {
+		if(noticeablyChanged) {
+			landmarkSmoothers[i * 2].reset();
+			landmarkSmoothers[i * 2 + 1].reset();
+		} else {
+			preds[i] = Vector2f(
+				landmarkSmoothers[i * 2].responsiveAnalogReadSimple(preds[i](0)),
+				landmarkSmoothers[i * 2 + 1].responsiveAnalogReadSimple(preds[i](1))
+			);
+		}
+	}
+
+	float minX = preds[0](0);
+	float maxX = preds[0](0);
+	float minY = preds[0](1);
+	float maxY = preds[0](1);
+	float sumX = preds[0](0);
+	float sumY = preds[0](1);
+
+	for (size_t i = 1; i < LANDMARK_POINT_COUNT; i++)
+	{
+		if (minX > preds[i](0))
+		{
+			minX = preds[i](0);
+		}
+
+		if (minY > preds[i](1))
+		{
+			minY = preds[i](1);
+		}
+
+		if (maxX < preds[i](0))
+		{
+			maxX = preds[i](0);
+		}
+
+		if (maxY < preds[i](1))
+		{
+			maxY = preds[i](1);
+		}
+
+		sumX += preds[i](0);
+		sumY += preds[i](1);
+	}
+
+	float avgX = sumX / static_cast<float>(LANDMARK_POINT_COUNT);
+	float avgY = sumY / static_cast<float>(LANDMARK_POINT_COUNT);
+
+	std::vector<cv::Point2f> imagePoints;
+	imagePoints.push_back(cv::Point2f(preds[17](0), preds[17](1))); // #17 left brow left corner
+	imagePoints.push_back(cv::Point2f(preds[21](0), preds[21](1))); // #21 left brow right corner
+	imagePoints.push_back(cv::Point2f(preds[22](0), preds[22](1))); // #22 right brow left corner
+	imagePoints.push_back(cv::Point2f(preds[26](0), preds[26](1))); // #26 right brow right corner
+	imagePoints.push_back(cv::Point2f(preds[36](0), preds[36](1))); // #36 left eye left corner
+	imagePoints.push_back(cv::Point2f(preds[39](0), preds[39](1))); // #39 left eye right corner
+	imagePoints.push_back(cv::Point2f(preds[42](0), preds[42](1))); // #42 right eye left corner
+	imagePoints.push_back(cv::Point2f(preds[45](0), preds[45](1))); // #45 right eye right corner
+	imagePoints.push_back(cv::Point2f(preds[31](0), preds[31](1))); // #31 nose left corner
+	imagePoints.push_back(cv::Point2f(preds[35](0), preds[35](1))); // #35 nose right corner
+	imagePoints.push_back(cv::Point2f(preds[48](0), preds[48](1))); // #48 mouth left corner
+	imagePoints.push_back(cv::Point2f(preds[54](0), preds[54](1))); // #54 mouth right corner
+	imagePoints.push_back(cv::Point2f(preds[57](0), preds[57](1))); // #57 mouth central bottom corner
+	imagePoints.push_back(cv::Point2f(preds[8](0), preds[8](1)));   // #8 chin corner
+
+	cv::solvePnP(modelPoints, imagePoints, cameraMatrix, distortionCoefficients, rotationVector, translationVector);
+
+	//if(similarity > 0.01) {
+	//	for (size_t i = 0; i < 3; i++) {
+	//		rotationVectorSmoothers[i].reset();
+	//		translationVectorSmoothers[i].reset();
+	//	}
+	//}
+
+	/*for(size_t i = 0; i < 3; i++) {
+		rotationVector.at<double>(i) = rotationVectorSmoothers[i].responsiveAnalogReadSimple(rotationVector.at<double>(i));
+		 translationVector.at<double>(i) = translationVectorSmoothers[i].responsiveAnalogReadSimple(translationVector.at<double>(i));
+	}*/
+
+	cv::projectPoints(boxModelSrc, rotationVector, translationVector, cameraMatrix, distortionCoefficients, boxModelDst);
+
+	// calc Euler angles
+	cv::Rodrigues(rotationVector, rotationMatrix);
+	cv::hconcat(rotationMatrix, translationVector, poseMatrix);
+
+	cv::Mat outIntrinsics = cv::Mat(3, 3, CV_64FC1);
+	cv::Mat outRotation = cv::Mat(3, 3, CV_64FC1);
+	cv::Mat outTranslation = cv::Mat(3, 1, CV_64FC1);
+	cv::decomposeProjectionMatrix(poseMatrix, outIntrinsics, outRotation, outTranslation, cv::noArray(), cv::noArray(), cv::noArray(), eulerAngles);
+
+	center = {avgX, avgY};
+	size = {maxX- minX, maxY - minY};
+}
+
+void FaceStruct::ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients)
+{
+	InnerApplyPreds(fromPreds);
+	CalcFromPreds(noticeablyChanged, cameraMatrix, distortionCoefficients);
+}
+
+void FaceStruct::InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds)
+{
+	for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+	{
+		preds[i] = { fromPreds[i].x, 720 - fromPreds[i].y};
+		
+	}	
+}
+
+cv::VideoCapture cap;
+
+FaceLandmarkDetector faceLandmarkDetector;
+
+GLuint lastFrameTexture;
+
+std::array<FaceLandmarkStruct, MAX_FACE_COUNT> faceLandmarkArr;
+
+cv::Mat frame;
+
+std::vector<cv::Mat> frameArr;
+
+int currentIndex = -1;
+int lastProcessedIndex = 0;
+int lineShift = 0;
+
+int prevPassedIndex = 0;
+
+int indexStep = 0;
+
+std::array<FaceStruct, MAX_FACE_COUNT> faceStruct;
+
+
+const std::string CONST_LOADING_BACKGROUND_BLACK = "loading_background_black";
+
+
+TMyApplication* Application;
 
 
 void TMyApplication::InnerInit()
@@ -103,32 +214,91 @@ void TMyApplication::InnerInit()
     }
     srand (static_cast<size_t>(time(NULL)));
 
-	ResourceManager->ShaderManager.AddShader("AntialiasingShader", "ms_vertex.txt", "ms_fragment.txt");
+	ResourceManager->ShaderManager.AddShader("DefaultShader", "shader1vertex.txt", "shader1fragment.txt");
+	ResourceManager->ShaderManager.AddShader("FrameShader", "frameshader_vertex.txt", "frameshader_fragment.txt");
+	Renderer->PushShader("DefaultShader");
+
+	ResourceManager->TexList.AddTexture(CONST_LOADING_BACKGROUND_BLACK + ".png", CONST_LOADING_BACKGROUND_BLACK);
 
 
-	ResourceManager->TexList.AddTexture("console_bkg.bmp");
+	auto texid = ResourceManager->TexList.AddTexture("emoji/Asset 3@20x.png");
-
+	ResourceManager->TexList.AddTexture("emoji/Asset 4@20x.png");
-	std::vector<std::pair<Vector2f, Vector2f>> lineArr;
+	ResourceManager->TexList.AddTexture("emoji/Asset 5@20x.png");
-
+	ResourceManager->TexList.AddTexture("emoji/Asset 6@20x.png");
-	for (int i = 0; i < 10; i++)
+	ResourceManager->TexList.AddTexture("emoji/Asset 7@20x.png");
-	{
+	ResourceManager->TexList.AddTexture("emoji/Asset 8@20x.png");
-		lineArr.push_back({ {50.f + i * 20.f, 50.f}, { 60.f + i * 20.f, 440.f } });
+	ResourceManager->TexList.AddTexture("emoji/Asset 9@20x.png");
-
+	ResourceManager->TexList.AddTexture("emoji/Asset 10@20x.png");
-		lineArr.push_back({ { 60.f + i * 20.f, 440.f },{ 70.f + i * 20.f, 50.f } });
+	ResourceManager->TexList.AddTexture("emoji/Asset 11@20x.png");
-	};
+	ResourceManager->TexList.AddTexture("emoji/Asset 13@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 14@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 15@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 16@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 17@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 18@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 19@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 20@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 21@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 22@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 23@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 24@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 25@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 26@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 27@20x.png");
+	ResourceManager->TexList.AddTexture("emoji/Asset 28@20x.png");
 
 
+	ResourceManager->FontManager.AddFont("droid_sans14", "droid_sans14_font_bitmap.png", "droid_sans14_font_charmap.txt");
+	ResourceManager->FontManager.PushFont("droid_sans14");
 
-	lineRenderPair = linesToRenderBoxes(lineArr);
 
-	Renderer->PushShader("AntialiasingShader");
+	lastFrameTexture = ResourceManager->TexList.AddEmptyTexture("lastFrameTexture", 1280, 720);
 
 	Renderer->SetOrthoProjection();
 
 	Renderer->SetFullScreenViewport();
 
-	Inited = true;
 
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		faceRenderPair[i].first.SamplerMap[CONST_STRING_TEXTURE_UNIFORM] = "Asset "+boost::lexical_cast<std::string>(i+3)+"@20x.png";
+
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 0, 0));
+		faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0));
+
+
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 0));
+		faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0));
+		faceRenderPair[i].second.RefreshBuffer();
+	}
+
+
+	cap = cv::VideoCapture(0);
+	//cap = cv::VideoCapture("video.mp4");
+	//cap = cv::VideoCapture("bp.mp4");
+
+
+	// Check if camera opened successfully
+	if (!cap.isOpened()) {
+		std::cout << "Error opening video stream or file" << std::endl;
+	}
+
+	cap.set(CV_CAP_PROP_FRAME_WIDTH, 1280);
+	cap.set(CV_CAP_PROP_FRAME_HEIGHT, 720);
+
+
+		
+	faceLandmarkDetector.StartTrackProcess();
+
+	Inited = true;
 
 
 
@@ -137,20 +307,18 @@ void TMyApplication::InnerInit()
 
 void TMyApplication::InnerDeinit()
 {
+
+	faceLandmarkDetector.StopTrackProcess();
+	cap.release();
+
     Inited = false;
-    Loaded = false;
+    
     if (Console != NULL)
     {
         *Console<<"APP DEINIT\n";
     }
-}
 
-void TMyApplication::OnMouseWheel(short int delta)
+
-{
-	scale += (delta > 0 ? 1 : -1) * 0.25;
-	if(scale < 0.25) {
-		scale = 0.25;
-	}
 }
 
 void TMyApplication::InnerOnTapDown(Vector2f p)
@@ -178,23 +346,26 @@ void TMyApplication::OnFling(Vector2f v)
 }
 
 
+	
 void TMyApplication::InnerDraw()
 {
+	glDisable(GL_DEPTH_TEST);
 
-	//Render the frame
-	Renderer->SetProjectionMatrix(512.f, 512.f);
-	Renderer->PushMatrix();
-	Renderer->ScaleMatrix(scale);
 
-	glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
-	glClear(GL_DEPTH_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
-	CheckGlError("");
 
-	TRenderParamsSetter params(lineRenderPair.first);
+	//glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]);
-	Renderer->DrawTriangleList(lineRenderPair.second);
+	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
+	Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight()));
+
+
+	/*
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		TRenderParamsSetter params(faceRenderPair[i].first);
+		Renderer->DrawTriangleList(faceRenderPair[i].second);
+	}*/
+	
 
-	Renderer->PopMatrix();
-	CheckGlError("");
 
 
 }
@@ -202,13 +373,187 @@ void TMyApplication::InnerDraw()
 
 void TMyApplication::InnerUpdate(size_t dt)
 {
+	cap >> frame; // get a new frame from camera/video or read image
 
+	if (frame.empty())
+	{
+		return;
+	}
+
+	cv::Mat image = frame.clone();
+	
+	currentIndex++;
+
+	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
+
+#ifdef USE_PREDICTION
+
+	frameArr.push_back(image);
+
+	while (frameArr.size() > 40)
+	{
+		lineShift++;
+		frameArr.erase(frameArr.begin());
+	}
+
+	currentIndex++;
+
+	faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex);
+
+
+	bool newProcessedIndexFound = false;
+	int newProcessedIndex;
+
+	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
+	{
+		if (faceLandmarkArr[i].valid)
+		{
+			if (!newProcessedIndexFound)
+			{
+				newProcessedIndexFound = true;
+				newProcessedIndex = faceLandmarkArr[i].frameIndex;
+			}
+			else
+			{
+				if (newProcessedIndex < faceLandmarkArr[i].frameIndex)
+				{
+					newProcessedIndex = faceLandmarkArr[i].frameIndex;
+				}
+			}
+		}
+	}
+
+
+	if (newProcessedIndexFound && newProcessedIndex > lastProcessedIndex)
+	{
+		indexStep = 0;
+
+		lastProcessedIndex = newProcessedIndex;
+		prevPassedIndex = currentIndex;
+
+	}
+	else
+	{
+		if (prevPassedIndex > lastProcessedIndex + indexStep)
+		{
+			indexStep++;
+		}
+	}
+
+	//std::cout << currentIndex << " " << lastProcessedIndex << " stepped:" << lastProcessedIndex + indexStep << " indexStep: " << indexStep << std::endl;
+
+
+	cv::Mat renderImage;
+
+	if (lastProcessedIndex + indexStep - lineShift < 0)
+	{
+		renderImage = frameArr[0];
+	}
+	else if (lastProcessedIndex + indexStep - lineShift >= frameArr.size())
+	{
+		renderImage = frameArr[frameArr.size() - 1];
+	}
+	else
+	{
+		renderImage = frameArr[lastProcessedIndex + indexStep - lineShift];
+	}
+
+	renderImage = renderImage.clone();
+
+#else
+	cv::Mat renderImage = image; 
+#endif
+
+	cv::Mat cameraMatrix = (cv::Mat_<float>(3, 3) << 9.5434798118121421e+02, 0.0, 640.0, 0.0, 9.5434798118121421e+02, 360.0, 0.0, 0.0, 1.0);
+	cv::Mat distortionCoefficients = (cv::Mat_<float>(5, 1) << 3.5475494652047600e-02, - 5.9818881494543774e-01, 0.0, 0.0, 1.4642154802454528e+00);
+
+	for (size_t i = 0; i < faceLandmarkArr.size(); i++)
+	{
+		if (faceLandmarkArr[i].valid)
+		{
+			cv::rectangle(renderImage, faceLandmarkArr[i].faceRect, cv::Scalar(0, 0, 255), 1, 4, 0);
+
+			for (size_t j = 0; j < faceLandmarkArr[i].landmarkArr.size(); j++)
+			{
+
+				//cv::circle(renderImage, faceLandmarkArr[i].landmarkArr[j], 0.1, cv::Scalar(0, 255, 255), 4, 8, 0);
+
+			}
+
+		}
+	}
+
+	for (size_t index = 0; index < faceLandmarkArr.size(); index++)
+	{
+		if (faceLandmarkArr[index].valid)
+		{
+			faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr, faceLandmarkArr[index].noticeablyChanged, cameraMatrix, distortionCoefficients);
+
+			for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++)
+			{
+				cv::ellipse(renderImage, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)), cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0);
+			}
+
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[0]), flipVertical(faceStruct[index].boxModelDst[1]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[1]), flipVertical(faceStruct[index].boxModelDst[2]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[2]), flipVertical(faceStruct[index].boxModelDst[3]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[3]), flipVertical(faceStruct[index].boxModelDst[0]), cv::Scalar(0, 255, 0));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[4]), flipVertical(faceStruct[index].boxModelDst[5]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[5]), flipVertical(faceStruct[index].boxModelDst[6]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[6]), flipVertical(faceStruct[index].boxModelDst[7]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[7]), flipVertical(faceStruct[index].boxModelDst[4]), cv::Scalar(0, 255, 0));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[0]), flipVertical(faceStruct[index].boxModelDst[4]), cv::Scalar(0, 255, 0));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[1]), flipVertical(faceStruct[index].boxModelDst[5]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[2]), flipVertical(faceStruct[index].boxModelDst[6]), cv::Scalar(0, 0, 255));
+			cv::line(renderImage, flipVertical(faceStruct[index].boxModelDst[3]), flipVertical(faceStruct[index].boxModelDst[7]), cv::Scalar(0, 255, 0));
+
+			Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.35f;
+			Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.35f;
+
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+		else
+		{
+			ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], { 0,0 }, { 0,0 }, 0);
+
+			faceRenderPair[index].second.RefreshBuffer();
+
+		}
+
+	}
+
+
+	glBindTexture(GL_TEXTURE_2D, lastFrameTexture);
+
+	static std::array<char, 1280 * 720 * 3> buffer;
+
+	int step = renderImage.step;
+	int height = renderImage.rows;
+	int width = renderImage.cols;
+	int channels = 3;
+	char * data = (char *)renderImage.data;
+	for (int i = 0; i < height; i++)
+	{
+		int ci = height - i - 1;
+		memcpy(&buffer[i*width*channels], &(data[ci*step]), width*channels);
+	}
+
+
+	glTexImage2D(
+		GL_TEXTURE_2D,
+		0,
+		GL_RGB,
+		1280,
+		720,
+		0,
+		GL_BGR_EXT,
+		GL_UNSIGNED_BYTE,
+		&buffer[0]);
 }
 
-bool TMyApplication::IsLoaded()
-{
-    return Loaded;
-}
 
 bool TMyApplication::IsInited()
 {

game/main_code.h

@@ -24,22 +24,56 @@
 
 #include "boost/signals2.hpp"//Signal must be included after asio
 
+#include "../windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h"
+#include "../windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h"
+
+
+struct FaceStruct
+{
+	SE::Vector2f center;
+	SE::Vector2f size;
+
+	std::array<SE::Vector2f, LANDMARK_POINT_COUNT> preds;
+
+	std::vector<cv::Point3f> modelPoints;
+	cv::Mat rotationVector;
+	cv::Mat translationVector;
+	cv::Mat rotationMatrix;
+	cv::Mat poseMatrix;
+	cv::Mat eulerAngles;
+
+	std::vector<cv::Point3d> boxModelSrc;
+	std::vector<cv::Point2d> boxModelDst;
+
+	ValueSmoother landmarkSmoothers[2 * LANDMARK_POINT_COUNT];
+	ValueSmoother rotationVectorSmoothers[3];
+	ValueSmoother translationVectorSmoothers[3];
+
+	FaceStruct();
+	FaceStruct(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
+
+	void CalcFromPreds(bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficients);
+	void ApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPred, bool noticeablyChanged, cv::Mat& cameraMatrix, cv::Mat& distortionCoefficientss);
+	void InnerApplyPreds(const std::array<cv::Point2f, LANDMARK_POINT_COUNT>& fromPreds);
+};
+
+
+
 using namespace SE;
 
+extern const std::string CONST_LOADING_BACKGROUND_BLACK;
+
 class TMyApplication : public TApplication
 {
 protected:
 
 	bool Inited;
 
-	float scale = 1.0;
+	std::array<TRenderPair, MAX_FACE_COUNT> faceRenderPair;
 
 public:
-	TRenderPair lineRenderPair;
 
-	bool Loaded;
+	TMyApplication() : TApplication(), Inited(false) { }
-
-	TMyApplication() : TApplication(), Loaded(false), Inited(false) { }
 
 	virtual void InnerInit();
 
@@ -49,12 +83,8 @@ public:
 
 	virtual void InnerUpdate(size_t dt);
 
-	bool IsLoaded();
-
 	bool IsInited();
 
-	virtual void OnMouseWheel(short int delta);
-
 	virtual void InnerOnTapDown(Vector2f p);
 
 	virtual void InnerOnTapUp(Vector2f p);
@@ -73,3 +103,5 @@ extern TMyApplication* Application;
 
 
 #endif
+
+

game/menucode.cpp

@@ -1,220 +0,0 @@
-#include "menucode.h"
-#include "main_code.h"
-
-const float CONST_SPEED_NEG_ACCELERATION_K = 0.03f;
-
-const float CONST_MENU_POS_K = 0.25f;
-
-const float CONST_MENU_WINDOW_DISTANCE = 240.f + 40.f;
-
-const float CONST_SHIFT_ON_SIDE = 0.f;
-
-TGameMenu::TGameMenu()
-	: MenuSpeed(0)
-	, MenuPos(0)
-	, HoldToTap(false)
-	, MenuItemCount(1)
-	, SelectedGame(-1)
-{
-}
-
-
-void TGameMenu::Draw()
-{
-//	*SE::Console << "TGameMenu::Draw";
-    CheckGlError("Draw TGameMenu");
-    RenderUniform1i("sel", 0);
-    RenderUniform1f("Transparency", 1.f);
-    float bkgShift = MenuPos*0.1f - 100.f;
-	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["main_menu_bkg_left"]);	
-	Renderer->DrawRect(Vector2f(bkgShift,0.f), Vector2f(480.f+bkgShift,320.f));
-	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["main_menu_bkg_right"]);
-	Renderer->DrawRect(Vector2f(480.f+bkgShift,0.f), Vector2f(960.f+bkgShift,320.f));
-
-
-	if (SelectedGame == 0)
-	{
-        RenderUniform1i("sel", 1);
-    }
-    else
-    {
-        RenderUniform1i("sel", 0);
-    }
-    
-    for (int i=0; i<MenuItemCount; i++)
-    {
-        glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["levelshot"+tostr(i+1)]);
-        Renderer->DrawRect(Vector2f(160.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*i,80.f), Vector2f(160.f+240.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*i, 240.f));
-    }
-    
-    if (MenuItemCount == 12)
-    {
-        glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["game_end"]);
-        Renderer->DrawRect(Vector2f(160.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*12,160.f - 64.f), Vector2f(160.f+256.f+MenuPos+CONST_MENU_WINDOW_DISTANCE*12, 160.f + 64.f));
-    
-    }
-    
-    
-    
-    glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["select_level"]);
-    Renderer->DrawRect(Vector2f(240.f-128.f, 241.f), Vector2f(240.f+128.f, 305.f));
-    
-	glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList["logo_small"]);
-    Renderer->DrawRect(Vector2f(240.f-128.f+15.f, 0.f), Vector2f(240.f+128.f+15.f, 64.f));
-
-    CheckGlError("Draw TGameMenu 2");
-
-	
-}
-
-void TGameMenu::Update(size_t dt)
-{
-	if (HoldToTap)
-		return;
-	// åñëè êíîïêà íå íàæàòà
-	float k;
-	
-	MenuPos = MenuPos + MenuSpeed*dt/1000.f;
-
-    int menuItemToShowCount;
-    
-    if (MenuItemCount == 12)
-    {
-        menuItemToShowCount = 13;
-    }
-    else
-    {
-        menuItemToShowCount = MenuItemCount;
-    }
-	
-	float acc;
-	
-	if (MenuItemCount == 1)
-	{
-	   k = 3;
-    }
-	else
-	{
-        k = fabs( - MenuPos - ((menuItemToShowCount-1)*CONST_MENU_WINDOW_DISTANCE*0.5f)) / ((MenuItemCount-1)*CONST_MENU_WINDOW_DISTANCE*0.5f);  
-    }
-    
-	acc = -MenuSpeed * CONST_SPEED_NEG_ACCELERATION_K * k;
-	
-    
-	float xleft = (- MenuPos + CONST_SHIFT_ON_SIDE - (menuItemToShowCount-1)*CONST_MENU_WINDOW_DISTANCE);
-	
-	if (xleft < 0)
-	{
-        xleft = 0;
-    }
-    
-    acc += xleft*CONST_MENU_POS_K;
-    
-    float xright = (MenuPos - CONST_SHIFT_ON_SIDE);
-	
-	if (xright < 0)
-	{
-        xright = 0;
-    }
-    
-    acc -= xright*CONST_MENU_POS_K;
-    
-	if ((MenuSpeed + acc*dt) * MenuSpeed < 0) //Check if MenuSpeed changes direction
-	{
-		MenuSpeed = 0;
-	}
-	else
-	{
-		MenuSpeed += acc*dt;
-	}
-    
-    
-}
-
-void TGameMenu::OnTapDown(Vector2f pos)
-{
-	*SE::Console << "TGameMenu::OnTapDown";
-    HoldToTap = true;
-	
-    if (pos(1)<64.f && pos(0)>=265.f-128.f && pos(0)<=265.f+128.f)
-    {
-        Application->GoFromMenuToCredits();
-        return;
-    }
-	
-    Vector2f realPos = pos - Vector2f(MenuPos, 0);
-    
-    if (realPos(1) >= 80.f && realPos(1) <= 240.f)
-    {
-        float x = realPos(0) - 160.f;
-        
-        int p = 0;
-        while (x >280.f)
-        {
-            x -= 280.f;
-            p++;
-        }
-        
-        if (x <= 240.f && x >= 0.f && (p<MenuItemCount))
-        {
-            SelectedGame = p;
-            return;
-        }
-    }
-    
-
-    SelectedGame = -1;
-
-}
-
-void TGameMenu::OnTapUp(Vector2f pos)
-{
-	*SE::Console << "TGameMenu::OnTapUp";
-	HoldToTap = false;
-	
-	if (SelectedGame != -1)
-	{
-        MenuPos = -CONST_MENU_WINDOW_DISTANCE*SelectedGame-40.f;
-        Application->GoFromMenuToGame(SelectedGame);
-        SelectedGame = -1;
-    }
-}
-
-void TGameMenu::OnTapUpAfterMove(Vector2f pos)
-{
-    HoldToTap = false;
-}
-
-
-void TGameMenu::OnFling(Vector2f slideSpeed)
-{
-	*SE::Console << "TGameMenu::OnFling";
-	HoldToTap = false;
-	MenuSpeed = slideSpeed(0);
-}
-
-void TGameMenu::OnScroll(Vector2f shift)
-{
-	MenuPos = MenuPos - shift(0);
-}
-
-void TGameMenu::OpenNextLevel()
-{
-	*SE::Console << "TGameMenu::OpenNextLevel";
-    if (MenuItemCount < 12)
-    {
-        MenuItemCount++;
-    }
-}
-
-int TGameMenu::GetMenuItemCount()
-{
-	*SE::Console << "TGameMenu::GetMenuItemCount";
-    return MenuItemCount;
-}
-
-void TGameMenu::SetMenuItemCount(int menuItemCount)
-{
-	*SE::Console << "TGameMenu::SetMenuItemCount";
-    MenuItemCount = menuItemCount;
-}

game/menucode.h

@@ -1,41 +0,0 @@
-#ifndef MENUCODE_H_INCLUDED
-#define MENUCODE_H_INCLUDED
-
-#include "include/Engine.h"
-#include "game_area_interface.h"
-
-using namespace SE;
-
-
-class TGameMenu : public TGameAreaAncestor
-{
-protected:
-	float MenuSpeed;
-	float MenuPos;
-	
-	bool HoldToTap;
-	
-	int MenuItemCount;
-	
-	int SelectedGame;
-	
-	
-public:
-	TGameMenu();
-	virtual void Draw();
-	virtual void Update(size_t dt);
-	virtual void OnTapDown(Vector2f pos);
-	virtual void OnTapUp(Vector2f pos);
-    virtual void OnTapUpAfterMove(Vector2f pos);
-	virtual void OnFling(Vector2f slideSpeed);
-	virtual void OnScroll(Vector2f shift);
-	
-	void OpenNextLevel();
-	int GetMenuItemCount();
-	void SetMenuItemCount(int menuItemCount);
-
-};
-
-
-
-#endif

windows/DoubleHitBalls-win/DoubleHitBalls-win/DoubleHitBalls-win.vcxproj

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
       <Configuration>Debug</Configuration>
@@ -22,27 +22,28 @@
     <ProjectGuid>{1CC98EEE-BBCB-4D79-B6D7-8511789172C5}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>
     <RootNamespace>DoubleHitBallswin</RootNamespace>
-    <WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
+    <WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
+    <ProjectName>TorchFaceTracker</ProjectName>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
     <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
     <WholeProgramOptimization>true</WholeProgramOptimization>
     <CharacterSet>NotSet</CharacterSet>
-    <PlatformToolset>v140</PlatformToolset>
+    <PlatformToolset>v141</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
@@ -111,13 +112,13 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>Disabled</Optimization>
       <PreprocessorDefinitions>TARGET_WIN32;DEBUG;_WIN32_WINNT=0x0501;EIGEN_DONT_ALIGN_STATICALLY;_WINDOWS;WIN32;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_63_0/;../../../../boost_1_63_0/bin.v2/libs/system/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/date_time/build/msvc-14.0/debug/address-model-64/link-static/threading-multi/;../../../../boost_1_63_0/bin.v2/libs/regex/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/thread/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/chrono/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/signals/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>../../../../luajit-vlad/luajit-vlad;..\..\..\..\..\Torch7\distro\install\include;C:\Work\TorchProjects\dlib;D:\Work\TorchProjects\dlib\dlib\..;..\..\..\..\..\Torch7\distro\exe\luajit-2.1\src;..\..\..\..\..\opencv3.4\build\include;D:\Work\Torch7\distro\install\include;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
       <DisableSpecificWarnings>4503</DisableSpecificWarnings>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_63_0/stage/x64/lib;../../../../boost_1_63_0/bin.v2/libs/system/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/date_time/build/msvc-14.0/debug/address-model-64/link-static/threading-multi/;../../../../boost_1_63_0/bin.v2/libs/regex/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/thread/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/chrono/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../boost_1_63_0/bin.v2/libs/signals/build/msvc-14.0/debug/address-model-64/link-static/threading-multi;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Debug</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Debug</AdditionalLibraryDirectories>
       <AdditionalDependencies>libjpeg.lib;libpng.lib;Engine.lib;opengl32.lib;glu32.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
@@ -147,11 +148,11 @@
       <WarningLevel>Level3</WarningLevel>
       <PrecompiledHeader>
       </PrecompiledHeader>
-      <Optimization>MaxSpeed</Optimization>
+      <Optimization>Disabled</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;TARGET_WIN32;_WIN32_WINNT=0x0501;WIN32_LEAN_AND_MEAN;EIGEN_DONT_ALIGN_STATICALLY;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>../../../../luajit-vlad/luajit-vlad;..\..\..\..\..\Torch7\distro\install\include;C:\Work\TorchProjects\dlib;D:\Work\TorchProjects\dlib\dlib\..;..\..\..\..\..\Torch7\distro\exe\luajit-2.1\src;..\..\..\..\..\opencv3.4\build\include;D:\Work\Torch7\distro\install\include;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510</AdditionalIncludeDirectories>
       <DisableSpecificWarnings>4503</DisableSpecificWarnings>
     </ClCompile>
     <Link>
@@ -159,17 +160,21 @@
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
-      <AdditionalLibraryDirectories>../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/boost-gil-extension;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../../../../libs/lpng1510;../x64/Release</AdditionalLibraryDirectories>
+      <AdditionalLibraryDirectories>..\..\..\..\..\opencv3.4-build\lib\$(Configuration);..\..\..\..\..\opencv3.4-build\3rdparty\lib\$(Configuration);..\..\..\..\..\opencv3.4-build\3rdparty\ippicv\ippicv_win\lib\intel64;../../../game;../../../../tes-engine;../../../../eigen;../../../../boost_1_67_0/stage/lib;../../../../libs/jpeg-9;../../../../libs/jpeg-9/vc10;../x64/Release;C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.1/lib/x64;C:/Program Files/NVIDIA GPU Computing</AdditionalLibraryDirectories>
-      <AdditionalDependencies>libjpeg.lib;libpng.lib;Engine.lib;opengl32.lib;glu32.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>ippiw.lib;ippicvmt.lib;cudart.lib;nppc.lib;nppial.lib;nppicc.lib;nppicom.lib;nppidei.lib;nppif.lib;nppig.lib;nppim.lib;nppist.lib;nppisu.lib;nppitc.lib;npps.lib;cufft.lib;opencv_dnn341.lib;opencv_highgui341.lib;opencv_videoio341.lib;opencv_imgcodecs341.lib;opencv_imgproc341.lib;opencv_core341.lib;opencv_calib3d341.lib;opencv_cudev341.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;comdlg32.lib;advapi32.lib;Engine.lib;opengl32.lib;glu32.lib;libjpeg-static.lib;libpng-static.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\game\main_code.cpp" />
+    <ClCompile Include="FaceLandmarkDetector.cpp" />
     <ClCompile Include="main.cpp" />
+    <ClCompile Include="ValueSmoother.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\game\main_code.h" />
+    <ClInclude Include="FaceLandmarkDetector.h" />
     <ClInclude Include="main.h" />
+    <ClInclude Include="ValueSmoother.h" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">

windows/DoubleHitBalls-win/DoubleHitBalls-win/DoubleHitBalls-win.vcxproj.filters

@@ -21,6 +21,12 @@
     <ClCompile Include="..\..\..\game\main_code.cpp">
       <Filter>Файлы исходного кода</Filter>
     </ClCompile>
+    <ClCompile Include="FaceLandmarkDetector.cpp">
+      <Filter>Файлы исходного кода</Filter>
+    </ClCompile>
+    <ClCompile Include="ValueSmoother.cpp">
+      <Filter>Файлы исходного кода</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="main.h">
@@ -29,5 +35,11 @@
     <ClInclude Include="..\..\..\game\main_code.h">
       <Filter>Заголовочные файлы</Filter>
     </ClInclude>
+    <ClInclude Include="FaceLandmarkDetector.h">
+      <Filter>Заголовочные файлы</Filter>
+    </ClInclude>
+    <ClInclude Include="ValueSmoother.h">
+      <Filter>Заголовочные файлы</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

windows/DoubleHitBalls-win/DoubleHitBalls-win/DoubleHitBalls-win.vcxproj.user

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup />
+</Project>

windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.cpp

@@ -0,0 +1,490 @@
+#include "FaceLandmarkDetector.h"
+
+static const size_t faceTrackerWidth = 300;
+static const size_t faceTrackerHeight = 300;
+
+static const double faceTrackerScaleFactor = 1.0;
+static const cv::Scalar faceTrackerMeanVal(104.0, 177.0, 123.0);
+
+float calcSquaredDistanceBetweenFaces(const cv::Rect& face1, const cv::Rect& face2)
+{
+	cv::Point2f oldCenter(face1.x + face1.width*0.5f, face1.y + face1.height*0.5f);
+	cv::Point2f newCenter(face2.x + face2.width*0.5f, face2.y + face2.height*0.5f);
+
+	float distanceX = newCenter.x - oldCenter.x;
+	float distanceY = newCenter.y - oldCenter.y;
+
+	float squareDistance = distanceX * distanceX + distanceY * distanceY;
+
+	return squareDistance;
+}
+
+double calcSimilarity(const cv::Mat& first, const cv::Mat& second)
+{
+	// assumes that first and second have the same size
+	double errorL2 = cv::norm(first, second, CV_L2);
+	// Convert to a reasonable scale, since L2 error is summed across all pixels of the image.
+	double similarity = errorL2 / static_cast<double>(first.rows * first.cols);
+	return similarity;
+}
+
+FaceLandmarkDetector::FaceLandmarkDetector()
+{
+
+
+	static const cv::String faceTrackerConfiguration = "deploy.prototxt";
+	static const cv::String faceTrackerBinary = "res10_300x300_ssd_iter_140000.caffemodel";
+
+
+#ifdef USE_3D_LANDMARKS
+	static const cv::String landmarkTrackerConfiguration = "fa_deploy.prototxt";
+	static const cv::String landmarkTrackerBinary = "fa__iter_1400000.caffemodel";
+
+#else
+	static const cv::String landmarkTrackerConfiguration = "landmark_deploy.prototxt";
+	static const cv::String landmarkTrackerBinary = "VanFace.caffemodel";
+
+#endif
+
+	faceNet = cv::dnn::readNetFromCaffe(faceTrackerConfiguration, faceTrackerBinary);
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		landmarkNetArr[i] = cv::dnn::readNetFromCaffe(landmarkTrackerConfiguration, landmarkTrackerBinary);
+	}
+}
+
+FaceLandmarkDetector::~FaceLandmarkDetector()
+{
+	StopTrackProcess();
+}
+
+
+void FaceLandmarkDetector::StartTrackProcess()
+{
+	if (started)
+	{
+		return;
+	}
+
+
+	//Some init setup
+
+	syncStop = false;
+
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		syncFaceRectArr[i].valid = false;
+		syncFaceLandmarkArr[i].valid = false;
+	}
+
+	syncLastFrame = cv::Mat();
+
+
+	//Start threads
+
+	findFaceThread = std::thread([this]() { this->FindFaceThreadProc(); });
+
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		landmarkThreadArr[i] = std::thread([this, i]() { this->FindLandmarkThreadProc(i); });
+	}
+
+	started = true;
+
+}
+
+void FaceLandmarkDetector::StopTrackProcess()
+{
+	if (!started)
+	{
+		return;
+	}
+
+	//Do not remove braces
+	{
+		std::unique_lock<std::mutex> lock(syncMutex);
+
+		syncStop = true;
+
+		syncFaceRectCond.notify_all();
+	}
+
+
+	findFaceThread.join();
+
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		landmarkThreadArr[i].join();
+	}
+
+	started = false;
+
+}
+
+std::array<FaceLandmarkStruct, MAX_FACE_COUNT> FaceLandmarkDetector::GetFaceLandmarks(cv::Mat frame, int frameIndex)
+{
+
+	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> result;
+	//Sync area
+
+	std::unique_lock<std::mutex> lock(syncMutex);
+
+	// OUT to face rect thread >>>>
+	syncLastFrame = frame;
+
+	syncLastFrameIndex = frameIndex;
+
+	// IN from landmark threads <<<<
+	result = syncFaceLandmarkArr;
+
+	return result;
+}
+
+
+void FaceLandmarkDetector::FindFaceThreadProc()
+{
+	cv::Mat localFrame;
+	std::array<FaceIntermediateStruct, MAX_FACE_COUNT> localFaceRectArr;
+
+	bool localStop = false;
+
+	int localFrameIndex = 0;
+
+
+	while (!syncStop)
+	{
+
+		if (!localFrame.empty())
+		{
+			std::vector<cv::Rect> faces = InnerFindFaces(localFrame);
+			std::vector<cv::Mat> faceFrames;
+
+			//Get rid of faces that are too big
+			for (size_t i = 0; i < faces.size(); )
+			{
+				if (faces[i].x < 0 || faces[i].y < 0 || (faces[i].x + faces[i].width >= localFrame.cols) || (faces[i].y + faces[i].height >= localFrame.rows))
+				{
+					faces.erase(faces.begin() + i);
+				}
+				else
+				{
+					i++;
+				}
+			}
+
+			//Cut faces from frame
+			for (size_t i = 0; i <faces.size(); i++)
+			{
+				faceFrames.push_back(InnerCutFaceFrame(localFrame, faces[i]));
+			}
+
+			InnerSeparateFacesToBuckets(faces, faceFrames, localFaceRectArr);
+
+
+			for (size_t i = 0; i < localFaceRectArr.size(); i++)
+			{
+				if (localFaceRectArr[i].valid)
+				{
+					localFaceRectArr[i].frameIndex = localFrameIndex;
+				}
+			}
+		}
+
+
+
+		//Sync area
+
+		std::unique_lock<std::mutex> lock(syncMutex);
+
+		// OUT to landmark Threads >>>>
+		syncFaceRectArr = localFaceRectArr;
+
+		syncFaceRectCond.notify_all();
+
+
+		// IN from main thread <<<<
+		localFrame = syncLastFrame;
+
+		localFrameIndex = syncLastFrameIndex;
+
+		localStop = syncStop;
+	}
+}
+
+void FaceLandmarkDetector::FindLandmarkThreadProc(size_t threadIndex)
+{
+	FaceIntermediateStruct localInputFace;
+	FaceLandmarkStruct localOutputLandmarks;
+
+	bool localStop = false;
+	while (!localStop)
+	{
+		if (localInputFace.valid)
+		{
+#ifdef USE_3D_LANDMARKS
+			cv::Mat landmarkInputBlob = cv::dnn::blobFromImage(localInputFace.faceFrame, 1.0, cv::Size(40, 40), cv::Scalar(), false, false);
+#else
+			cv::Mat landmarkInputBlob = cv::dnn::blobFromImage(localInputFace.faceFrame, 1.0, cv::Size(60, 60), cv::Scalar(), false, false);
+#endif
+			
+			landmarkInputBlob.reshape(0, std::vector<int>{ 1, 1, localInputFace.faceFrame.rows, localInputFace.faceFrame.cols });
+
+			landmarkNetArr[threadIndex].setInput(landmarkInputBlob);
+#ifdef USE_3D_LANDMARKS
+			cv::Mat featureBlob = landmarkNetArr[threadIndex].forward("Dense2");
+#else
+			cv::Mat featureBlob = landmarkNetArr[threadIndex].forward("Dense3");
+#endif
+
+			for (int i = 0; i < featureBlob.cols / 2; i++)
+			{
+				cv::Point landmarkPoint;
+
+				landmarkPoint.x = int(featureBlob.at<float>(0, 2 * i) * (localInputFace.faceRect.width) + localInputFace.faceRect.x);
+				landmarkPoint.y = int(featureBlob.at<float>(0, 2 * i + 1) * (localInputFace.faceRect.height) + localInputFace.faceRect.y);
+				localOutputLandmarks.landmarkArr[i] = landmarkPoint;
+					
+			}
+
+			localOutputLandmarks.faceRect = localInputFace.faceRect;
+			localOutputLandmarks.frameIndex = localInputFace.frameIndex;
+			localOutputLandmarks.noticeablyChanged = localInputFace.noticeablyChanged;
+			localOutputLandmarks.valid = true;
+		}
+		else
+		{
+			localOutputLandmarks.valid = false;
+		}
+
+
+
+		//Sync area
+		std::unique_lock<std::mutex> lock(syncMutex);
+
+		// OUT to main thread >>>>
+		syncFaceLandmarkArr[threadIndex] = localOutputLandmarks;
+
+		bool doLoop = true;
+
+		while (doLoop)
+		{
+
+			//IN from face rect thread <<<<
+
+			localInputFace = syncFaceRectArr[threadIndex];
+
+
+			localStop = syncStop;
+
+			//Time to quit
+			if (localStop)
+			{
+				doLoop = false;
+			}
+
+			if (!localInputFace.valid)
+			{
+
+				syncFaceLandmarkArr[threadIndex].valid = false;
+
+				syncFaceRectCond.wait(lock);
+
+			}
+			else
+			{
+				doLoop = false;
+			}
+		}
+
+	}
+}
+
+
+std::vector<cv::Rect> FaceLandmarkDetector::InnerFindFaces(cv::Mat frame)
+{
+	if (frame.channels() == 4)
+	{
+		cvtColor(frame, frame, cv::COLOR_BGRA2BGR);
+	}
+
+	cv::Mat inputBlob = cv::dnn::blobFromImage(frame, faceTrackerScaleFactor, cv::Size(faceTrackerWidth, faceTrackerHeight), faceTrackerMeanVal, false, false); 
+
+	faceNet.setInput(inputBlob, "data");
+
+	cv::Mat detection = faceNet.forward("detection_out");
+
+	cv::Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+
+	float confidenceThreshold = 0.5f;
+
+	std::vector<cv::Rect> faces;
+
+	for (int i = 0; i < detectionMat.rows; i++)
+	{
+		float confidence = detectionMat.at<float>(i, 2);
+
+		if (confidence > confidenceThreshold)
+		{
+			int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frame.cols);
+			int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frame.rows);
+			int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frame.cols);
+			int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frame.rows);
+
+			cv::Rect object((int)xLeftBottom, (int)yLeftBottom,
+				(int)(xRightTop - xLeftBottom),
+				(int)(yRightTop - yLeftBottom));
+
+
+			object.x = object.x - object.width*0.1f;
+			object.y = object.y - object.height*0.1f;
+			object.width = object.width*1.2f;
+			object.height = object.height*1.2f;
+
+			faces.push_back(object);
+		}
+	}
+
+	return faces;
+
+}
+
+cv::Mat FaceLandmarkDetector::InnerCutFaceFrame(cv::Mat frame, cv::Rect faceRect)
+{
+
+	
+	cv::Mat srcROI(frame, faceRect);
+
+	cv::Mat img2;
+	cvtColor(srcROI, img2, CV_RGB2GRAY);
+
+	img2.convertTo(img2, CV_32FC1);
+#ifdef USE_3D_LANDMARKS
+	cv::Size dsize = cv::Size(40, 40);
+#else
+	cv::Size dsize = cv::Size(60, 60);
+#endif
+	
+	cv::Mat img3 = cv::Mat(dsize, CV_32FC1);
+	cv::resize(img2, img3, dsize, 0, 0, cv::INTER_CUBIC);
+
+	cv::Mat tmp_m, tmp_sd;
+	double m = 0, sd = 0;
+	meanStdDev(img3, tmp_m, tmp_sd);
+	m = tmp_m.at<double>(0, 0);
+	sd = tmp_sd.at<double>(0, 0);
+
+	img3 = (img3 - m) / (0.000001 + sd);
+
+	return img3;
+}
+
+void FaceLandmarkDetector::InnerSeparateFacesToBuckets(std::vector<cv::Rect>& faceArr, std::vector<cv::Mat>& faceFrameArr, std::array<FaceIntermediateStruct, MAX_FACE_COUNT>& inOutBucketArr)
+{
+	static std::array<std::vector<std::pair<cv::Rect, cv::Mat>>, MAX_FACE_COUNT> facesBelongsToSpace;
+
+
+	for (size_t i = 0; i < MAX_FACE_COUNT; i++)
+	{
+		facesBelongsToSpace[i].clear();
+	}
+
+
+	//Distribute current faces to nearest old positions
+	for (size_t i = 0; i < faceArr.size(); )
+	{
+		float lastDistance = 1000000.f;
+		int lastIndex = -1;
+
+		for (size_t j = 0; j < MAX_FACE_COUNT; j++)
+		{
+			if (inOutBucketArr[j].valid)
+			{
+				float newDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, faceArr[i]);
+				if (newDistance < lastDistance)
+				{
+					lastDistance = newDistance;
+					lastIndex = j;
+				}
+			}
+		}
+
+		if (lastIndex != -1)
+		{
+			facesBelongsToSpace[lastIndex].push_back(std::make_pair(faceArr[i], faceFrameArr[i]));
+			faceArr.erase(faceArr.begin() + i);
+			faceFrameArr.erase(faceFrameArr.begin() + i);
+		}
+		else
+		{
+			i++;
+		}
+	}
+
+	//Mark "free spots" to those that don't have any face belong to
+	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
+	{
+		if (inOutBucketArr[j].valid)
+		{
+			if (facesBelongsToSpace[j].size() == 0)
+			{
+				inOutBucketArr[j].valid = false;
+			}
+		}
+	}
+
+	//Find nearest face in each space - others return back to pool
+	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
+	{
+		if (inOutBucketArr[j].valid)
+		{
+			float lastDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, facesBelongsToSpace[j][0].first);
+			int lastIndex = 0;
+
+			for (size_t i = 1; i < facesBelongsToSpace[j].size(); i++)
+			{
+				float newDistance = calcSquaredDistanceBetweenFaces(inOutBucketArr[j].faceRect, facesBelongsToSpace[j][i].first);
+				if (newDistance < lastDistance)
+				{
+					lastDistance = newDistance;
+					lastIndex = i;
+				}
+			}
+
+			inOutBucketArr[j].faceRect = facesBelongsToSpace[j][lastIndex].first;
+			inOutBucketArr[j].faceFrame = facesBelongsToSpace[j][lastIndex].second;
+			inOutBucketArr[j].noticeablyChanged =
+				calcSimilarity(inOutBucketArr[j].faceFrame, facesBelongsToSpace[j][lastIndex].second) > 0.01 ||
+				lastDistance > 100.0;
+
+			facesBelongsToSpace[j].erase(facesBelongsToSpace[j].begin() + lastIndex);
+
+			for (size_t k = 0; k < facesBelongsToSpace[j].size(); k++)
+			{
+				faceArr.push_back(facesBelongsToSpace[j][k].first);
+				faceFrameArr.push_back(facesBelongsToSpace[j][k].second);
+			}
+		}
+	}
+
+	//Distribute faces from pool to empty slots
+	for (size_t j = 0; j < MAX_FACE_COUNT; j++)
+	{
+		if (!inOutBucketArr[j].valid)
+		{
+
+			if (faceArr.size() > 0)
+			{
+				inOutBucketArr[j].valid = true;
+
+				inOutBucketArr[j].faceRect = faceArr[0];
+				inOutBucketArr[j].faceFrame = faceFrameArr[0];
+				
+
+				faceArr.erase(faceArr.begin());
+				faceFrameArr.erase(faceFrameArr.begin());
+			}
+		}
+	}
+
+
+}

windows/DoubleHitBalls-win/DoubleHitBalls-win/FaceLandmarkDetector.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include <opencv2/core.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/calib3d.hpp>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <chrono>
+#include <iostream>
+
+#include <opencv2/dnn.hpp>
+
+#include <thread>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+
+#define USE_3D_LANDMARKS
+
+//#include "boost/thread.hpp"
+
+constexpr size_t LANDMARK_POINT_COUNT = 68;
+constexpr size_t MAX_FACE_COUNT = 5;
+
+struct FaceIntermediateStruct
+{
+	bool valid = false;
+
+	cv::Rect faceRect;
+	cv::Mat faceFrame;
+
+	int frameIndex;
+	bool noticeablyChanged = false;
+};
+
+
+struct FaceLandmarkStruct
+{
+	bool valid = false;
+
+	cv::Rect faceRect;
+	std::array<cv::Point2f, LANDMARK_POINT_COUNT> landmarkArr;
+
+	int frameIndex;
+	bool noticeablyChanged = false;
+};
+
+
+class FaceLandmarkDetector
+{
+public:
+	FaceLandmarkDetector();
+	~FaceLandmarkDetector();
+
+	void StartTrackProcess();
+
+	void StopTrackProcess();
+
+	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> GetFaceLandmarks(cv::Mat frame, int frameIndex);
+
+protected:
+	cv::dnn::Net faceNet;
+	
+	std::array<cv::dnn::Net, MAX_FACE_COUNT> landmarkNetArr;
+
+	bool started = false;
+
+
+	//Sync variables - shared between threads:
+
+	cv::Mat syncLastFrame;
+
+	int syncLastFrameIndex;
+
+	std::array<FaceIntermediateStruct, MAX_FACE_COUNT> syncFaceRectArr;
+
+	std::array<FaceLandmarkStruct, MAX_FACE_COUNT> syncFaceLandmarkArr;
+
+	std::mutex syncMutex;
+
+	bool syncStop = false;
+
+	std::condition_variable syncFaceRectCond;
+
+	std::thread findFaceThread;
+	void FindFaceThreadProc();
+
+	std::array<std::thread, MAX_FACE_COUNT> landmarkThreadArr;
+	void FindLandmarkThreadProc(size_t threadIndex);
+
+
+
+	std::vector<cv::Rect> InnerFindFaces(cv::Mat frame);
+
+	cv::Mat InnerCutFaceFrame(cv::Mat frame, cv::Rect faceRect);
+
+	void InnerSeparateFacesToBuckets(std::vector<cv::Rect>& faceArr, std::vector<cv::Mat>& faceFrameArr, std::array<FaceIntermediateStruct, MAX_FACE_COUNT>& inOutBucketArr);
+};

windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.cpp

@@ -0,0 +1,98 @@
+#include "ValueSmoother.h"
+
+
+#include <array>
+#include <chrono>
+
+using namespace std;
+
+using namespace std::chrono;
+
+
+float snapCurve(float x)
+{
+	float y = 1.0 / (x + 1.0);
+	y = (1.0 - y) * 2.0;
+	if (y > 1.0) {
+		return 1.0;
+	}
+	return y;
+}
+
+
+
+void ValueSmoother::reset()
+{
+	inited = false;
+}
+
+
+float ValueSmoother::responsiveAnalogReadSimple(float newValue)
+{
+	if (!inited)
+	{
+		inited = true;
+		smoothValue = newValue;
+		return smoothValue;
+	}
+	auto diff = abs(newValue - smoothValue);
+	float snap = snapCurve(diff * SNAP_MULTIPLIER_SIMPLE);
+
+	smoothValue += (newValue - smoothValue) * snap;
+	return smoothValue;
+}
+
+
+
+
+
+
+void ExtendedValueSmoother::reset()
+{
+	inited = false;
+}
+
+float ExtendedValueSmoother::responsiveAnalogRead(float newValue)
+{
+	using namespace std::chrono;
+
+	auto ms = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+
+
+	auto diff = abs(newValue - smoothValue);
+
+	// measure the difference between the new value and current value over time
+	// to get a more reasonable indication of how far off the current smooth value is
+	// compared to the actual measurements
+	errorEMA += ((newValue - smoothValue) - errorEMA) * 0.4;
+
+	// if sleep has been enabled, keep track of when we're asleep or not by marking the time of last activity and testing to see how much time has passed since then
+	if (SLEEP_ENABLE) {
+		// recalculate sleeping status
+		// (asleep if last activity was over SLEEP_DELAY_MS ago)
+		sleeping = abs(errorEMA) < ACTIVITY_THRESHOLD;
+	}
+
+
+	if (SLEEP_ENABLE && sleeping) {
+		return smoothValue;
+	}
+
+
+
+	auto snap = snapCurve(diff * SNAP_MULTIPLIER);
+
+	// when sleep is enabled, the emphasis is stopping on a responsiveValue quickly, and it's less about easing into position. If sleep is enabled, add a small amount to snap so it'll tend to snap into a more accurate position before sleeping starts.
+	if (SLEEP_ENABLE) {
+		snap = snap * 0.5 + 0.5;
+	}
+
+	// calculate the exponential moving average based on the snap
+	smoothValue += (newValue - smoothValue) * snap;
+
+	return smoothValue;
+
+}
+
+
+

windows/DoubleHitBalls-win/DoubleHitBalls-win/ValueSmoother.h

@@ -0,0 +1,49 @@
+#pragma once
+
+
+
+class ValueSmoother {
+public:
+	
+
+	void reset();
+
+	float responsiveAnalogReadSimple(float newValue);
+
+protected:
+	bool inited = false;
+
+	float SNAP_MULTIPLIER_SIMPLE = 1.0 / 60;
+
+	//float SNAP_MULTIPLIER_SIMPLE = 0.3;
+
+	float smoothValue = 0;
+
+};
+
+
+struct ExtendedValueSmoother {
+public:
+	void reset();
+
+	float responsiveAnalogRead(float newValue);
+
+protected:
+
+	bool inited = false;
+
+	float SNAP_MULTIPLIER = 1.0 / 200;
+
+	float smoothValue = 0;
+
+	bool SLEEP_ENABLE = true;
+
+	float ACTIVITY_THRESHOLD = 16.0;
+
+	float errorEMA = 0;
+
+	long long lastActivityMS = 0;
+
+	bool sleeping = false;
+
+};

windows/DoubleHitBalls-win/DoubleHitBalls-win/fa_deploy.prototxt

@@ -0,0 +1,293 @@
+name: "CNNFacepoint"
+layer {
+	name: "data"
+	type: "Input"
+	top: "data"
+	input_param {
+		shape: {dim: 1 dim: 1 dim: 40 dim: 40}
+	}
+}
+
+layer {
+    name: "Conv1"
+    type: "Convolution"
+    bottom: "data"
+    top: "Conv1"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 16
+        pad: 2
+        kernel_size: 5
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+layer {
+  name: "ActivationTangH1"
+  bottom: "Conv1"
+  top: "ActivationTangH1"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs1"
+  bottom: "ActivationTangH1"
+  top: "Abs1"
+  type: "AbsVal"
+}
+
+layer {
+    name: "Pool1"
+    type: "Pooling"
+    bottom: "Abs1"
+    top: "Pool1"
+    pooling_param {
+        pool: MAX
+        kernel_size: 2
+        stride: 2
+    }
+}
+
+layer {
+    name: "Conv2"
+    type: "Convolution"
+    bottom: "Pool1"
+    top: "Conv2"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 48
+        pad: 1
+        kernel_size: 3
+        stride: 1
+        weight_filler {
+        type: "xavier"
+        std: 0.1
+    }
+    bias_filler {
+        type: "constant"
+        value: 0.2
+        }
+    }
+}
+
+layer {
+  name: "ActivationTangH2"
+  bottom: "Conv2"
+  top: "ActivationTangH2"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs2"
+  bottom: "ActivationTangH2"
+  top: "Abs2"
+  type: "AbsVal"
+}
+
+
+layer {
+    name: "Pool2"
+    type: "Pooling"
+    bottom: "Abs2"
+    top: "Pool2"
+    pooling_param {
+        pool: MAX
+        kernel_size: 2
+        stride: 2
+    }
+}
+
+# layer 3
+layer {
+    name: "Conv3"
+    type: "Convolution"
+    bottom: "Pool2"
+    top: "Conv3"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 64
+        pad: 0
+        kernel_size: 3
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH3"
+  bottom: "Conv3"
+  top: "ActivationTangH3"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs3"
+  bottom: "ActivationTangH3"
+  top: "Abs3"
+  type: "AbsVal"
+}
+
+layer {
+    name: "Pool3"
+    type: "Pooling"
+    bottom: "Abs3"
+    top: "Pool3"
+    pooling_param {
+        pool: MAX
+        kernel_size: 3
+        stride: 2
+    }
+}
+
+# layer 4
+layer {
+    name: "Conv4"
+    type: "Convolution"
+    bottom: "Pool3"
+    top: "Conv4"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 64
+        pad: 0
+        kernel_size: 2
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH4"
+  bottom: "Conv4"
+  top: "ActivationTangH4"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs4"
+  bottom: "ActivationTangH4"
+  top: "Abs4"
+  type: "AbsVal"
+}
+
+
+########################################
+
+layer {
+    name: "Dense1"
+    type: "InnerProduct"
+    bottom: "Abs4"
+    top: "Dense1"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    inner_product_param {
+        num_output: 100
+        weight_filler {
+            type: "xavier"
+        }
+        bias_filler {
+            type: "constant"
+            value: 0
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH5"
+  bottom: "Dense1"
+  top: "ActivationTangH5"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs5"
+  bottom: "ActivationTangH5"
+  top: "Abs5"
+  type: "AbsVal"
+}
+
+
+layer {
+    name: "Dense2"
+    type: "InnerProduct"
+    bottom: "Abs5"
+    top: "Dense2"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    inner_product_param {
+        num_output: 136
+        weight_filler {
+            type: "xavier"
+        }
+        bias_filler {
+            type: "constant"
+            value: 0
+        }
+    }
+}
+
+

windows/DoubleHitBalls-win/DoubleHitBalls-win/landmark_deploy.prototxt

@@ -0,0 +1,292 @@
+name: "landmark"
+input: "data"
+input_shape {
+  dim: 1
+  dim: 1
+  dim: 60
+  dim: 60
+}
+########################################
+# the actual net
+# layer 1
+layer {
+    name: "Conv1"
+    type: "Convolution"
+    bottom: "data"
+    top: "Conv1"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 20
+        pad: 2
+        kernel_size: 5
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+layer {
+  name: "ActivationTangH1"
+  bottom: "Conv1"
+  top: "ActivationTangH1"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs1"
+  bottom: "ActivationTangH1"
+  top: "Abs1"
+  type: "AbsVal"
+}
+
+layer {
+    name: "Pool1"
+    type: "Pooling"
+    bottom: "Abs1"
+    top: "Pool1"
+    pooling_param {
+        pool: MAX
+        kernel_size: 2
+        stride: 2
+    }
+}
+
+layer {
+    name: "Conv2"
+    type: "Convolution"
+    bottom: "Pool1"
+    top: "Conv2"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 48
+        pad: 2
+        kernel_size: 5
+        stride: 1
+        weight_filler {
+        type: "xavier"
+        std: 0.1
+    }
+    bias_filler {
+        type: "constant"
+        value: 0.2
+        }
+    }
+}
+
+layer {
+  name: "ActivationTangH2"
+  bottom: "Conv2"
+  top: "ActivationTangH2"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs2"
+  bottom: "ActivationTangH2"
+  top: "Abs2"
+  type: "AbsVal"
+}
+
+
+layer {
+    name: "Pool2"
+    type: "Pooling"
+    bottom: "Abs2"
+    top: "Pool2"
+    pooling_param {
+        pool: MAX
+        kernel_size: 2
+        stride: 2
+    }
+}
+
+# layer 3
+layer {
+    name: "Conv3"
+    type: "Convolution"
+    bottom: "Pool2"
+    top: "Conv3"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 64
+        pad: 0
+        kernel_size: 3
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH3"
+  bottom: "Conv3"
+  top: "ActivationTangH3"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs3"
+  bottom: "ActivationTangH3"
+  top: "Abs3"
+  type: "AbsVal"
+}
+
+layer {
+    name: "Pool3"
+    type: "Pooling"
+    bottom: "Abs3"
+    top: "Pool3"
+    pooling_param {
+        pool: MAX
+        kernel_size: 3
+        stride: 2
+    }
+}
+
+# layer 4
+layer {
+    name: "Conv4"
+    type: "Convolution"
+    bottom: "Pool3"
+    top: "Conv4"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    convolution_param {
+        num_output: 80
+        pad: 0
+        kernel_size: 3
+        stride: 1
+        weight_filler {
+            type: "xavier"
+            std: 0.1
+        }
+        bias_filler {
+            type: "constant"
+            value: 0.2
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH4"
+  bottom: "Conv4"
+  top: "ActivationTangH4"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs4"
+  bottom: "ActivationTangH4"
+  top: "Abs4"
+  type: "AbsVal"
+}
+
+
+########################################
+
+layer {
+    name: "Dense1"
+    type: "InnerProduct"
+    bottom: "Abs4"
+    top: "Dense1"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    inner_product_param {
+        num_output: 512
+        weight_filler {
+            type: "xavier"
+        }
+        bias_filler {
+            type: "constant"
+            value: 0
+        }
+    }
+}
+
+
+layer {
+  name: "ActivationTangH5"
+  bottom: "Dense1"
+  top: "ActivationTangH5"
+  type: "TanH"
+}
+
+layer {
+  name: "ActivationAbs5"
+  bottom: "ActivationTangH5"
+  top: "Abs5"
+  type: "AbsVal"
+}
+
+
+layer {
+    name: "Dense3"
+    type: "InnerProduct"
+    bottom: "Abs5"
+    top: "Dense3"
+    param {
+        lr_mult: 1
+        decay_mult: 1
+    }
+    param {
+        lr_mult: 2
+        decay_mult: 0
+    }
+    inner_product_param {
+        num_output: 136
+        weight_filler {
+            type: "xavier"
+        }
+        bias_filler {
+            type: "constant"
+            value: 0
+        }
+    }
+}

windows/DoubleHitBalls-win/DoubleHitBalls-win/main.cpp

@@ -4,12 +4,15 @@
 
 TMyApplication* App = NULL;
 
+
+
+
 int APIENTRY WinMain(HINSTANCE hCurrentInst, HINSTANCE hPreviousInst,
 	LPSTR lpszCmdLine, int nCmdShow)
 {
 
-	int width = 512;
+	int width = 1280;
-	int height = 512;
+	int height = 720;
 
 	
 	if (CreateEngine(width, height))
@@ -30,5 +33,6 @@ int APIENTRY WinMain(HINSTANCE hCurrentInst, HINSTANCE hPreviousInst,
 		DestroyEngine();
 	}
 
+	
 	return 0;
 }