#include "main_code.h" #include #include #include #include #include "include/Engine.h" #include "main_code.h" FaceStruct::FaceStruct() { } FaceStruct::FaceStruct(const std::array& fromPreds) { ApplyPreds(fromPreds); } void FaceStruct::CalcFromPreds() { float minX = preds[0](0); float maxX = preds[0](0); float minY = preds[0](1); float maxY = preds[0](1); float sumX = preds[0](0); float sumY = preds[0](1); for (size_t i = 1; i < LANDMARK_POINT_COUNT; i++) { if (minX > preds[i](0)) { minX = preds[i](0); } if (minY > preds[i](1)) { minY = preds[i](1); } if (maxX < preds[i](0)) { maxX = preds[i](0); } if (maxY < preds[i](1)) { maxY = preds[i](1); } sumX += preds[i](0); sumY += preds[i](1); } float avgX = sumX / static_cast(LANDMARK_POINT_COUNT); float avgY = sumY / static_cast(LANDMARK_POINT_COUNT); center = { avgX, avgY }; size = { maxX- minX,maxY - minY }; //center = { centerXSmoother.responsiveAnalogReadSimple(avgX), centerYSmoother.responsiveAnalogReadSimple(avgY) }; //size = { sizeWidthSmoother.responsiveAnalogReadSimple(maxX- minX), sizeHeightSmoother.responsiveAnalogReadSimple(maxY - minY) }; } void FaceStruct::ApplyPreds(const std::array& fromPreds) { InnerApplyPreds(fromPreds); CalcFromPreds(); /* for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { preds[i] = preds[i] - center; } historicalPreds.push_back(preds); if (historicalPreds.size() > 4) { historicalPreds.erase(historicalPreds.begin()); } for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { preds[i] = { 0.f, 0.f }; for (size_t x = 0; x < historicalPreds.size(); x++) { preds[i] += historicalPreds[x][i]; } preds[i] = preds[i] / static_cast(historicalPreds.size()); } for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { preds[i] = preds[i] + center; }*/ } void FaceStruct::InnerApplyPreds(const std::array& fromPreds) { for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { preds[i] = { fromPreds[i].x, 720 - fromPreds[i].y}; } } cv::VideoCapture cap; FaceLandmarkDetector faceLandmarkDetector; GLuint lastFrameTexture; std::array faceLandmarkArr; cv::Mat frame; std::vector frameArr; int currentIndex = -1; int lastProcessedIndex = 0; int lineShift = 0; int prevPassedIndex = 0; int indexStep = 0; std::array faceStruct; const std::string CONST_LOADING_BACKGROUND_BLACK = "loading_background_black"; TMyApplication* Application; void TMyApplication::InnerInit() { Application = this; #ifdef TARGET_WIN32 #ifdef NDEBUG //ST::PathToResources = "resources/"; ST::PathToResources = "../../../assets/"; #else ST::PathToResources = "../../../assets/"; #endif #endif #ifdef TARGET_IOS ST::PathToResources = "assets/"; #endif if (Console != NULL) { *Console<<"APP INIT\n"; } srand (static_cast(time(NULL))); ResourceManager->ShaderManager.AddShader("DefaultShader", "shader1vertex.txt", "shader1fragment.txt"); ResourceManager->ShaderManager.AddShader("FrameShader", "frameshader_vertex.txt", "frameshader_fragment.txt"); Renderer->PushShader("DefaultShader"); ResourceManager->TexList.AddTexture(CONST_LOADING_BACKGROUND_BLACK + ".png", CONST_LOADING_BACKGROUND_BLACK); auto texid = ResourceManager->TexList.AddTexture("emoji/Asset 3@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 4@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 5@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 6@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 7@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 8@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 9@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 10@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 11@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 13@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 14@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 15@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 16@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 17@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 18@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 19@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 20@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 21@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 22@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 23@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 24@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 25@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 26@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 27@20x.png"); ResourceManager->TexList.AddTexture("emoji/Asset 28@20x.png"); ResourceManager->FontManager.AddFont("droid_sans14", "droid_sans14_font_bitmap.png", "droid_sans14_font_charmap.txt"); ResourceManager->FontManager.PushFont("droid_sans14"); lastFrameTexture = ResourceManager->TexList.AddEmptyTexture("lastFrameTexture", 1280, 720); Renderer->SetOrthoProjection(); Renderer->SetFullScreenViewport(); for (size_t i = 0; i < MAX_FACE_COUNT; i++) { faceRenderPair[i].first.SamplerMap[CONST_STRING_TEXTURE_UNIFORM] = "Asset "+boost::lexical_cast(i+3)+"@20x.png"; faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0)); faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 512, 0)); faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0)); faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 512, 0)); faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(512, 0, 0)); faceRenderPair[i].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB].push_back(Vector3f(0, 0, 0)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 1)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 1)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(1, 0)); faceRenderPair[i].second.Data.Vec2CoordArr[CONST_STRING_TEXCOORD_ATTRIB].push_back(Vector2f(0, 0)); faceRenderPair[i].second.RefreshBuffer(); } cap = cv::VideoCapture(0); //cap = cv::VideoCapture("video.mp4"); //cap = cv::VideoCapture("bp.mp4"); // Check if camera opened successfully if (!cap.isOpened()) { std::cout << "Error opening video stream or file" << std::endl; } cap.set(CV_CAP_PROP_FRAME_WIDTH, 1280); cap.set(CV_CAP_PROP_FRAME_HEIGHT, 720); faceLandmarkDetector.StartTrackProcess(); Inited = true; } void TMyApplication::InnerDeinit() { faceLandmarkDetector.StopTrackProcess(); cap.release(); Inited = false; if (Console != NULL) { *Console<<"APP DEINIT\n"; } } void TMyApplication::InnerOnTapDown(Vector2f p) { } void TMyApplication::InnerOnTapUp(Vector2f p) { } void TMyApplication::InnerOnTapUpAfterMove(Vector2f p) { } void TMyApplication::InnerOnMove(Vector2f p, Vector2f shift) { } void TMyApplication::OnFling(Vector2f v) { } void TMyApplication::InnerDraw() { glDisable(GL_DEPTH_TEST); //glBindTexture(GL_TEXTURE_2D, ResourceManager->TexList[CONST_LOADING_BACKGROUND_BLACK]); glBindTexture(GL_TEXTURE_2D, lastFrameTexture); Renderer->DrawRect(Vector2f(0.f, 0.f), Vector2f(Renderer->GetMatrixWidth(), Renderer->GetMatrixHeight())); /* for (size_t i = 0; i < MAX_FACE_COUNT; i++) { TRenderParamsSetter params(faceRenderPair[i].first); Renderer->DrawTriangleList(faceRenderPair[i].second); }*/ } void TMyApplication::InnerUpdate(size_t dt) { /*cap >> lastFrame; if (lastFrame.empty()) { return; } faceLandmarkTracker.HandleOneFrame(lastFrame, localInputStructArr, localFaceShift, localOutputStructArr); high_resolution_clock::time_point t2 = high_resolution_clock::now(); auto duration = duration_cast(t2 - FaceLandmarkTracker::lastTimePoint).count(); FaceLandmarkTracker::lastTimePoint = t2; std::stringstream ofs; ofs << round(1000000.0 / duration); for (size_t index = 0; index < CONST_FACE_COUNT; index++) { if (localOutputStructArr[index].valid && localInputStructArr[index].valid) { cv::Rect oldFaceRect = localOutputStructArr[index].face; cv::Rect newFaceRect = localInputStructArr[index].face; cv::Point2f oldCenter(oldFaceRect.x + oldFaceRect.width*0.5f, oldFaceRect.y + oldFaceRect.height*0.5f); cv::Point2f newCenter(newFaceRect.x + newFaceRect.width*0.5f, newFaceRect.y + newFaceRect.height*0.5f); localFaceShift[index] = newCenter - oldCenter; } else { localFaceShift[index] = cv::Point2f(0, 0); } } for (size_t index = 0; index < CONST_FACE_COUNT; index++) { if (localOutputStructArr[index].valid) { faceStruct[index].ApplyPreds(localOutputStructArr[index].preds); for (size_t i = 0; i < CONST_PREDS_COUNT; i++) { //cv::ellipse(lastFrame, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)) + localFaceShift[index], cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0); } Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.25f; Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.25f; ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0); faceRenderPair[index].second.RefreshBuffer(); } else { ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], {0,0}, { 0,0 }, 0); faceRenderPair[index].second.RefreshBuffer(); } } for (size_t index = 0; index < CONST_FACE_COUNT; index++) { if (localInputStructArr[index].valid) { //cv::rectangle(lastFrame, localInputStructArr[index].face, cv::Scalar(255, 0, 255), 4, 8, 0); } } */ cap >> frame; // get a new frame from camera/video or read image if (frame.empty()) { return; } cv::Mat image = frame.clone(); frameArr.push_back(image); while (frameArr.size() > 40) { lineShift++; frameArr.erase(frameArr.begin()); } currentIndex++; faceLandmarkArr = faceLandmarkDetector.GetFaceLandmarks(frame, currentIndex); bool newProcessedIndexFound = false; int newProcessedIndex; for (size_t i = 0; i < faceLandmarkArr.size(); i++) { if (faceLandmarkArr[i].valid) { if (!newProcessedIndexFound) { newProcessedIndexFound = true; newProcessedIndex = faceLandmarkArr[i].frameIndex; } else { if (newProcessedIndex < faceLandmarkArr[i].frameIndex) { newProcessedIndex = faceLandmarkArr[i].frameIndex; } } } } if (newProcessedIndexFound && newProcessedIndex > lastProcessedIndex) { indexStep = 0; lastProcessedIndex = newProcessedIndex; prevPassedIndex = currentIndex; } else { if (prevPassedIndex > lastProcessedIndex + indexStep) { indexStep++; } } //std::cout << currentIndex << " " << lastProcessedIndex << " stepped:" << lastProcessedIndex + indexStep << " indexStep: " << indexStep << std::endl; cv::Mat renderImage; if (lastProcessedIndex + indexStep - lineShift < 0) { renderImage = frameArr[0]; } else if (lastProcessedIndex + indexStep - lineShift >= frameArr.size()) { renderImage = frameArr[frameArr.size() - 1]; } else { renderImage = frameArr[lastProcessedIndex + indexStep - lineShift]; } renderImage = renderImage.clone(); cv::Rect frameRect(cv::Point(), renderImage.size()); for (size_t i = 0; i < faceLandmarkArr.size(); i++) { if (faceLandmarkArr[i].valid) { cv::rectangle(renderImage, faceLandmarkArr[i].faceRect, cv::Scalar(0, 0, 255), 1, 4, 0); for (size_t j = 0; j < faceLandmarkArr[i].landmarkArr.size(); j++) { //cv::circle(renderImage, faceLandmarkArr[i].landmarkArr[j], 0.1, cv::Scalar(0, 255, 255), 4, 8, 0); } } } for (size_t index = 0; index < faceLandmarkArr.size(); index++) { if (faceLandmarkArr[index].valid) { faceStruct[index].ApplyPreds(faceLandmarkArr[index].landmarkArr); for (size_t i = 0; i < LANDMARK_POINT_COUNT; i++) { cv::ellipse(renderImage, cv::Point2f(faceStruct[index].preds[i](0), 720 - faceStruct[index].preds[i](1)), cv::Size(2, 2), 0, 0, 360, cv::Scalar(255, 128, 128), 4, 8, 0); } Vector2f posFrom = faceStruct[index].center - faceStruct[index].size*0.5f* 1.35f; Vector2f posTo = faceStruct[index].center + faceStruct[index].size*0.5f* 1.35f; ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], posFrom, posTo, 0); faceRenderPair[index].second.RefreshBuffer(); } else { ApplyVertexCoordVec(faceRenderPair[index].second.Data.Vec3CoordArr[CONST_STRING_POSITION_ATTRIB], { 0,0 }, { 0,0 }, 0); faceRenderPair[index].second.RefreshBuffer(); } } glBindTexture(GL_TEXTURE_2D, lastFrameTexture); static std::array buffer; int step = renderImage.step; int height = renderImage.rows; int width = renderImage.cols; int channels = 3; char * data = (char *)renderImage.data; for (int i = 0; i < height; i++) { int ci = height - i - 1; memcpy(&buffer[i*width*channels], &(data[ci*step]), width*channels); } glTexImage2D( GL_TEXTURE_2D, 0, GL_RGB, 1280, 720, 0, GL_BGR_EXT, GL_UNSIGNED_BYTE, &buffer[0]); } bool TMyApplication::IsInited() { return Inited; }