ofxFaceTracker mesh.drawFaces() drawing in wrong spot due to resolution

That title doesn’t even tell the story right.

Just to be clear of my specs:
I’m on OS X 10.11.1
OpenFrameworks v0.9.0
XCode 7.2

using these addons:
ofxCv
ofxFaceTracker
ofxGui
ofxKinect
ofxOpenCv
ofxPS3EyeGrabber

What I’ve done is taken the example-calibrated from the ofxFaceTracker addon and modified it so that instead of

cam.bind();
physicalMesh.drawFaces();
cam.unbind()

I’ve created my own ofImage and bound it to the faces. Also, I’m using a ps3 Eye Camera via the ofxPS3EyeGrabber addon (via @bakercp). Everything works fine except for that I am displaying the final image on a display screen that has a lowest resolution setting of 800x600. The ps3cam has a native and maximum of 640x480. So, I’ve made everything else sit in a window of 640x480 as a result. My ofImage is allocated 640x480, etc. Even the example-calibrated is all set to 640x480.

The problem is that when I scale everything to 800x600 the physicalMesh.drawFaces() draws as if it is still in a 640x480 window, missing the face it is supposed to draw on by whatever (-x, -y) difference.

Here is my code:

main.cpp

#include "ofAppGlutWindow.h"

int main() {
	ofAppGlutWindow window;
	ofSetupOpenGL(&window, 800, 600, OF_WINDOW); // i changed this from (640, 480)
	ofRunApp(new ofApp());
}

ofApp.h

#pragma once

#include "ofMain.h"
#include "ofxCv.h"
#include "ofxFaceTracker.h"
#include "ofxPS3EyeGrabber.h"


class ofApp : public ofBaseApp {
public:
	void setup();
	void updatePhysicalMesh();
	void update();
	void draw();
	void keyPressed(int key);
    void imageNoise();
    
    ofImage noise;
    int whiteBlack;
    int lineY;
    
	ofVideoGrabber ps3cam;
	ofxFaceTracker tracker;
	ofxCv::Calibration calibration;
	ofLight light;
	
	ofMesh physicalMesh;
    
    ofPixels pixelsRGB;
    
    bool swapBackground;
    
};

ofApp.cpp

#include "ofApp.h"

#include "ofMeshUtils.h"
#include "ofxPS3EyeGrabber.h"

using namespace ofxCv;
using namespace cv;

void ofApp::setup() {
	ofSetVerticalSync(true);
    
    // ps3EyeCamera
    cout << "elapsed time: " << ofGetElapsedTimeMillis() << endl;
    ps3cam.setGrabber(std::make_shared<ofxPS3EyeGrabber>());
    ps3cam.setDesiredFrameRate(45);
    ps3cam.setup(800, 600); // this is normally (640, 480)
    ps3cam.getGrabber<ofxPS3EyeGrabber>()->setAutogain(true);
    ps3cam.getGrabber<ofxPS3EyeGrabber>()->setAutoWhiteBalance(true);
    pixelsRGB.allocate(ps3cam.getWidth(), ps3cam.getHeight(), 3);
    
    // tracker
	tracker.setup();
	calibration.load("mbp-isight.yml");
	
	glLightModeli(GL_LIGHT_MODEL_TWO_SIDE, GL_TRUE);
	light.setPosition(100, 100, 1000);
    
    // noise
    noise.getPixels().allocate(800, 600,1); // i've changed this allocation from (640, 480)
    whiteBlack = 255;
    ofSetColor(255);
}

void ofApp::updatePhysicalMesh() {
	// 1 load object and image points as Point2f/3f
	vector<Point3f> objectPoints;
	vector<Point2f> imagePoints;
	for(int i = 0; i < tracker.size(); i++) {
		objectPoints.push_back(toCv(tracker.getObjectPoint(i)));
		imagePoints.push_back(toCv(tracker.getImagePoint(i)));
	}
	
	// 2 guess for the rotation and translation of the face
	Mat cameraMatrix = calibration.getDistortedIntrinsics().getCameraMatrix();
	Mat distCoeffs = calibration.getDistCoeffs();
	Mat rvec, tvec;
	solvePnP(Mat(objectPoints),  Mat(imagePoints),
					 cameraMatrix,  distCoeffs,
					 rvec, tvec);
	
	// 3 reproject using guess, and fit to the actual image location
	vector<ofVec3f> fitWorldPoints;
	Mat cameraMatrixInv = cameraMatrix.inv();
	Mat rmat;
	Rodrigues(rvec, rmat);
	for(int i = 0; i < objectPoints.size(); i++) {
		Point2d imgImg = imagePoints[i];
		Point3d objObj = objectPoints[i];
		Point3d imgHom(imgImg.x, imgImg.y, 1.); // img->hom
		Point3d imgWor = (Point3f) Mat(cameraMatrixInv * Mat(imgHom)); // hom->wor
		Point3d objWor = (Point3d) Mat(tvec + rmat * Mat(objObj)); // obj->wor
		Point3d fitWor = intersectPointRay(objWor, imgWor); // scoot it over
		// if it was projected on the wrong side, flip it over
		if(fitWor.z < 0) {
			fitWor *= -1;
		}
		fitWorldPoints.push_back(toOf(fitWor));
		// convert down to image space coordinates
		//Point3d fitHom = (Point3d) Mat(cameraMatrix * Mat(fitWor)); // wor->hom
		//Point2d fitImg(fitHom.x / fitHom.z, fitHom.y / fitHom.z); // hom->img
	}
	
	// 4 use the resulting 3d points to build a mesh with normals
	physicalMesh = convertFromIndices(tracker.getMesh(fitWorldPoints));
	physicalMesh.setMode(OF_PRIMITIVE_TRIANGLES);
	buildNormals(physicalMesh);
}

void ofApp::update() {
	ps3cam.update();
	
    if(ps3cam.isFrameNew()) {
        tracker.update(toCv(ps3cam));
        
        // face tracker mesh
		if(tracker.getFound()) {
			updatePhysicalMesh();
		}
	}
    
    // image noise
    lineY = ofRandom(ofGetHeight());
    whiteBlack = ofRandom(256);
    
    for (int x = 0; x < noise.getWidth(); ++x)
    {
        for (int y = 0; y < noise.getHeight(); ++y)
        {
            int index = noise.getPixels().getPixelIndex(x, y);
            
            if (y != lineY)
            {
                noise.getPixels()[index] = ofRandom(256);
            }
            else
            {
                noise.getPixels()[index] = whiteBlack;
            }
        }
    }
    noise.update();;
    
    // swap time and reset time
    if (ofGetMinutes() <= 15){
        swapBackground = false;
    }
    else if (ofGetMinutes() > 15 && ofGetMinutes() <= 30){
        swapBackground = true;
    }
    else if (ofGetMinutes() > 30 && ofGetMinutes() <= 45){
        swapBackground = false;
    }
    else if (ofGetMinutes() > 45 && ofGetMinutes() <= 59){
        swapBackground = true;
    }
    
    if (ofGetElapsedTimeMillis() >= 10000){
        tracker.reset();
        ofResetElapsedTimeCounter();
    }
    
    cout << "elapsed time: " << ofGetElapsedTimeMillis() << endl;
}

void ofApp::draw() {
    
    if (swapBackground == false){
        ps3cam.draw(0, 0, 800, 600); // drawing ps3cam from from 640x480 to 800x600
        // frames per second displayed
        std::stringstream ss;
        //
        //    ss << " App FPS: " << ofGetFrameRate() << std::endl;
        //    ss << " Cam FPS: " << ps3cam.getGrabber<ofxPS3EyeGrabber>()->getFPS()  << std::endl;
        //    ss << "Real FPS: " << ps3cam.getGrabber<ofxPS3EyeGrabber>()->getActualFPS() << std::endl;
        ss << "#ofNoiseFace" << std::endl;
        ss << "#openFrameworks" << std::endl;
        ss << "@mosspassion";
        
        ofDrawBitmapStringHighlight(ss.str(), ofPoint(10, 20));
    }
    else if (swapBackground == true){
        noise.draw(0, 0); // already allocated 800x600, no need to scale
        // frames per second displayed
        std::stringstream ss;
        //
        //    ss << " App FPS: " << ofGetFrameRate() << std::endl;
        //    ss << " Cam FPS: " << ps3cam.getGrabber<ofxPS3EyeGrabber>()->getFPS()  << std::endl;
        //    ss << "Real FPS: " << ps3cam.getGrabber<ofxPS3EyeGrabber>()->getActualFPS() << std::endl;
        ss << "#ofNoiseFace" << std::endl;
        ss << "#openFrameworks" << std::endl;
        ss << "@mosspassion";
        
        ofDrawBitmapStringHighlight(ss.str(), ofPoint(10, 20));
    }
    
    // tracker
	if(tracker.getFound()) {
        ofMesh objectMesh = tracker.getObjectMesh();
        ofMesh meanMesh = tracker.getMeanObjectMesh();
        
        ofSetupScreenOrtho(800, 600, -1000, 1000); // this was also (640, 480, -1000, 1000)
        ofScale(5,5,5);
        
		calibration.getDistortedIntrinsics().loadProjectionMatrix();

        // swap background and face
        if (swapBackground == false){
            ofEnableLighting();
            light.enable();
            noise.getTexture().bind();
            physicalMesh.drawFaces();
            noise.getTexture().unbind();
            ofDisableLighting();
        }
        
        else if (swapBackground == true){
            ps3cam.bind();
            physicalMesh.drawFaces();
            ps3cam.unbind();
        }
	}
}

void ofApp::keyPressed(int key) {
	if(key == 'r') {
		tracker.reset();
	}
    if (key == 'f'){
        ofToggleFullscreen();
    }
}

So , the result is this (on an external display, in OF_FULLSCREEN via ofToggleFullscreen()):


^That noise mask should be drawn right on my face^

My instructor, @bakercp, has led me to ofRectangle for scaling while retaining aspect ratio, and from the example /openFrameworks/examples/graphics/rectangleAlignmentAndScaling it looks like it is exactly capable of what I need, I just have no idea how to integrate it.

Any help would be appreciated, even any help on cleaning up my code from redundant things, anything. It’s almost done!

Thank you,
Nicholas

Hey, the code is a bit long and I don’t have PS3 Eye camera so I cannot test it, but it seems like you need

ofScale(800.0f/640.0f, 600.0f/480.0f);

before drawing the mesh.

1 Like

Hey thanks for responding… I found a different work around, but I will try the ofScale and check the reference for it. I really appreciate the help!