I have a project I am working on which involves five videos that get masked depending on how far away you are from a kinect. I would like to find better ways of working with the video frames, since this currently results in about 6 frames per second on my system. (videos are 700x411 (10fps), 1024 wide (6fps) 1600 wide (don’t ask))
Any help would be appreciated, but first, here is some code (forgive my unediting this):
#include "ofMain.h"
#include "testApp.h"
#include "ofAppGlutWindow.h"
//========================================================================
int main( ){
ofAppGlutWindow window;
// ofSetupOpenGL(&window, 700,411, OF_WINDOW); // <-------- setup the GL context
ofSetupOpenGL(&window, 700,411, OF_FULLSCREEN);
// this kicks off the running of my app
// can be OF_WINDOW or OF_FULLSCREEN
// pass in width and height too:
ofRunApp( new testApp());
}
#pragma once
#include "ofMain.h"
#include "ofxOpenCv.h"
#include "ofxKinect.h"
class testApp : public ofBaseApp{
public:
void setup();
void update();
void draw();
void exit();
void makeMasked(int threshold, ofVideoPlayer toBeMasked);
ofVideoPlayer cam1, cam2, cam3, cam4, cam5;
bool frameByframe;
int angle;
ofxKinect kinect;
/// used to switch between the live kinect and the recording player
ofxBase3DVideo* kinectSource;
ofxCvGrayscaleImage grayImage, grayedImage; // grayscale depth image
ofxCvGrayscaleImage grayThreshNear; // the near thresholded image
ofxCvGrayscaleImage grayThreshFar; // the far thresholded image
unsigned char * masked;
ofImage mask,mask1,makeMe;;
ofImage topLayer;
ofImage bottomLayer;
ofShader maskShader;
ofImage depthImage;
int HEIGHT, WIDTH;
};
#include "testApp.h"
//--------------------------------------------------------------
void testApp::setup(){
topLayer.loadImage("topLayer.png");
mask1.loadImage("blank.png");
bottomLayer.loadImage("bottomLayer.png");
frameByframe = true;
cam1.loadMovie("movies/cam1.mp4");
cam2.loadMovie("movies/cam2.mp4");
cam3.loadMovie("movies/cam3.mp4");
cam4.loadMovie("movies/cam4.mp4");
cam5.loadMovie("movies/cam5.mp4");
cam1.play();
cam2.play();
cam3.play();
cam4.play();
cam5.play();
// kinect.init();
kinect.init(true); // shows infrared instead of RGB video image
//kinect.init(false, false); // disable infrared/rgb video iamge (faster fps)
kinect.setVerbose(true);
kinect.open();
// start with the live kinect source
kinectSource = &kinect;
// zero the tilt on startup
angle = 10;
kinect.setCameraTiltAngle(angle);
grayImage.allocate(kinect.width, kinect.height);
grayedImage.allocate(kinect.width, kinect.height);
grayThreshNear.allocate(kinect.width, kinect.height);
grayThreshFar.allocate(kinect.width, kinect.height);
mask.allocate(kinect.width, kinect.height,OF_IMAGE_GRAYSCALE);
//need this for alpha to come through
ofEnableAlphaBlending();
// turn off vertical sync so we can evaluate performance
ofSetVerticalSync(false);
//set the texture parameters for the maks shader. just do this at the beginning
maskShader.load("composite");
maskShader.begin();
maskShader.setUniformTexture("Tex0", topLayer.getTextureReference(), 0);
maskShader.setUniformTexture("Tex1", mask1.getTextureReference(), 1);
maskShader.end();
HEIGHT = cam5.getHeight();
WIDTH = cam5.getWidth();
// this uses depth information for occlusion
// rather than always drawing things on top of each other
// glEnable(GL_DEPTH_TEST);
// the bind() function might complain about texture not
// being allocated, yet it seems to work fine. Uncomment
// the following line to stop the warning from displaying.
// ofSetLogLevel(OF_LOG_ERROR);
}
//--------------------------------------------------------------
void testApp::update(){
// fingerMovie.idleMovie();
cam1.idleMovie();
cam2.idleMovie();
cam3.idleMovie();
cam4.idleMovie();
cam5.idleMovie();
/*
cam1.setVolume(0);
cam2.setVolume(0);
cam3.setVolume(0);
cam4.setVolume(0);
cam5.setVolume(0);
*/
kinectSource->update();
// load grayscale depth image from the kinect source
grayedImage.setFromPixels(kinectSource->getDepthPixels(), kinect.width, kinect.height);
// kinect1Texture.loadData(kinectSource->getDepthPixels(), kinect.width, kinect.height,GL_RGB);
// mask = grayImage.getPixelsRef();
}
//--------------------------------------------------------------
void testApp::draw(){
// ofBackground(0, 0, 0);
cam5.draw(0,0, ofGetWidth(),ofGetHeight());
makeMasked(200, cam1);
makeMasked(150, cam2);
makeMasked(100, cam3);
makeMasked(50, cam5);
ofDrawBitmapString(ofToString(ofGetFrameRate()), 20, 20);
}
//--------------------------------------------------------------
void testApp::keyPressed (int key){
}
void testApp::makeMasked(int threshold, ofVideoPlayer toBeMasked){
grayImage = grayedImage.getPixels();
grayThreshNear = grayedImage.getPixels();
grayThreshFar = grayedImage.getPixels();
grayThreshNear.threshold(threshold,true);
grayThreshFar.threshold(threshold-60);
cvAnd(grayThreshNear.getCvImage(), grayThreshFar.getCvImage(), grayImage.getCvImage(), NULL);
grayImage.flagImageChanged();
mask = grayImage.getPixelsRef();
mask.resize(40, 30);
makeMe.setFromPixels(toBeMasked.getPixels(),WIDTH,HEIGHT,OF_IMAGE_COLOR);
mask.resize(WIDTH,HEIGHT);
//then draw a quad for the top layer using our composite shader to set the alpha
maskShader.begin();
//our shader uses two textures, the top layer and the alpha
//we can load two textures into a shader using the multi texture coordinate extensions
glActiveTexture(GL_TEXTURE0_ARB);
makeMe.getTextureReference().bind();
glActiveTexture(GL_TEXTURE1_ARB);
mask.getTextureReference().bind();
//draw a quad the size of the frame
glBegin(GL_QUADS);
glMultiTexCoord2d(GL_TEXTURE0_ARB, 0, 0);
glMultiTexCoord2d(GL_TEXTURE1_ARB, 0, 0);
glVertex2f( 0, 0);
glMultiTexCoord2d(GL_TEXTURE0_ARB, WIDTH, 0);
glMultiTexCoord2d(GL_TEXTURE1_ARB, WIDTH, 0);
glVertex2f( ofGetWidth(), 0);
glMultiTexCoord2d(GL_TEXTURE0_ARB, WIDTH, HEIGHT);
glMultiTexCoord2d(GL_TEXTURE1_ARB, WIDTH, HEIGHT);
glVertex2f( ofGetWidth(), ofGetHeight());
glMultiTexCoord2d(GL_TEXTURE0_ARB, 0, HEIGHT);
glMultiTexCoord2d(GL_TEXTURE1_ARB, 0, HEIGHT);
glVertex2f( 0, ofGetHeight());
glEnd();
//deactive and clean up
glActiveTexture(GL_TEXTURE1_ARB);
mask.getTextureReference().unbind();
//mask.unbind();
glActiveTexture(GL_TEXTURE0_ARB);
makeMe.getTextureReference().unbind();
maskShader.end();
}
//--------------------------------------------------------------
void testApp::exit() {
//kinect.close();
}
I should warn anyone trying this that it has a tendency to crash after loading the movies at “memcpy(thisKinect->videoPixelsBack, rgb, curMode.bytes);” in kinect::grabRgbFrame, and upon exiting with escape you will always get “malloc: *** error for object 0x7060707: pointer being freed was not allocated
*** set a breakpoint in malloc_error_break to debug”…
Any help (or suggestions for making it better) is appreciated. My fallback is to only play one movie depending on how close you are, but the silhouette is kind of cool.