blob tracker + motion mask.....+ shape?

Hi,

I need some help with how to combine tracked blobs with a motion mask. I have figured out how to play one video and fill the motion mask with a second video (this was a big accomplishment for me! I used code from: http://forum.openframeworks.cc/t/additive-motion-memory-for-video-masking./2378/0 to help) and now I also have another h file that I can draw the tracked blobs on top of a video, but I can’t figure out how to make this all work together. Ideally, what I really want is to have one video layer playing underneath, then fill the tracked blobs and detected motion with pixels from a second video clip.

Can I add the tracked blobs to the motion mask? Or, how can I also fill the blobs with the video pixels from the second video?

Also, the very last thing I want to do is:

The video clip underneath is of the ocean with the sun overhead, I want to draw the second video (which has reflection of light on the water) into the space where the viewers are standing (tracked blob) or moving (motion mask) but then… I would really like to also draw a line shape from the farthest right and farthest left pixel of each blob to a fixed point under where the sun is in the video (like the line of light we see reflected on the water when the sun is setting). So, can I turn the tracked blobs into a shape, with a vertex point under the sun? AND combine them to the motion mask?

Please see attached photo of sketch of what I ultimately want to accomplish.

If you can help me and give any suggestions or recommendations of examples to look at I would really, really appreciate it; VERY much. :slight_smile:
Thank You.

Also here are the two programs I have so far

  
#include "testApp.h"  
  
//////////////////////////////////////////////  
//	SETUP.  
//////////////////////////////////////////////  
  
void testApp :: setup()  
{  
	ofSetVerticalSync( true ); //i don't know what this is  
	ofSetFrameRate( 60 );//frame rate of program i think  
	ofBackground( 255, 255,255 );//background color  
	  
	initVideoTwo();//second video  
	initCamera();//camera feed  
	initVideoMask();//video mask  
	  
	initVideo();//first video  
	initMotion();//motion  
	  
	  
	  
}  
  
void testApp :: initCamera()//this is catching the camera  
{  
	cameraWidth		= 640;  
	cameraHeight	= 480;  
	  
	camera.setVerbose( true );  
	camera.initGrabber( cameraWidth, cameraHeight );  
}  
  
void testApp :: initVideo()//this is the first video  
{  
	video.loadMovie( "watersunreflect.mov" );  
	//unsigned char * pixels = video.getPixels();  
	//video.setLoopState(OF_LOOP_NORMAL);  
	video.play();  
}  
  
void testApp :: initVideoTwo()//this is the second video  
{  
	videotwo.loadMovie( "watersun640_480.mov" );  
	//unsigned char * pixels = videotwo.getPixels();  
	//videotwo.setLoopState(OF_LOOP_NORMAL);  
	videotwo.play();  
	  
}  
  
  
void testApp :: initMotion()//this is the motion  
{  
	cameraColorImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayPrevImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayDiffImage.allocate( cameraWidth, cameraHeight );  
	cameraDiffFloatImage.allocate( cameraWidth, cameraHeight );  
	cameraMotionFloatImage.allocate( cameraWidth, cameraHeight );  
	  
	cameraMotionFadeAmount = .95f;//allows for slow fade of motion drawn  
}  
  
void testApp :: initVideoMask()//this is the mask  
{  
	maskTexture.allocate( cameraWidth, cameraHeight, GL_RGBA );  
	  
	maskPixles = new unsigned char[ cameraWidth * cameraHeight * 4 ];  
}  
  
//////////////////////////////////////////////  
//	UPDATE.  
//////////////////////////////////////////////  
  
void testApp :: update()  
{  
	updateCamera();  
	  
	if( cameraNewFrame )  
	{  
		updateMotion( camera.getPixels() );  
		updateVideoMask();  
	}  
	  
		  
	  
	  
}  
  
void testApp :: updateCamera() //get the new camera frame  
{  
	camera.grabFrame();  
	cameraNewFrame = camera.isFrameNew();  
}  
  
void testApp :: updateMotion( unsigned char *pixels )//catch the new motion  
{  
	cameraColorImage.setFromPixels( pixels, cameraWidth, cameraHeight );  
	  
	cameraGrayPrevImage	= cameraGrayImage;  
	cameraGrayImage		= cameraColorImage;  
	  
	cameraGrayDiffImage.absDiff( cameraGrayImage, cameraGrayPrevImage );  
	cameraGrayDiffImage.threshold( 30 );  
	  
	cameraDiffFloatImage	= cameraGrayDiffImage;  
	cameraMotionFloatImage	*= cameraMotionFadeAmount;  
	cameraMotionFloatImage	+= cameraDiffFloatImage;  
	cameraMotionFloatImage.blur( 20 );  
}  
  
void testApp :: updateVideoMask()    
{  
	videoPixels		= video.getPixels();//whatever is in here is what fills the mask  
	motionPixles	= cameraMotionFloatImage.getPixels();  
	  
	for( int i=0; i<cameraWidth; i++ )  
	{  
		for( int j=0; j<cameraHeight; j++ )  
		{  
			int p = j * cameraWidth + i;  
			maskPixles[ p * 4 + 0 ] = videoPixels[ p * 3 + 0 ];   
			maskPixles[ p * 4 + 1 ] = videoPixels[ p * 3 + 1 ];   
			maskPixles[ p * 4 + 2 ] = videoPixels[ p * 3 + 2 ];   
			maskPixles[ p * 4 + 3 ] = motionPixles[ p ];  
		}  
	}  
	  
	maskTexture.loadData( maskPixles, cameraWidth, cameraHeight, GL_RGBA );  
}  
  
//////////////////////////////////////////////  
//	DRAW.  
//////////////////////////////////////////////  
  
void testApp :: draw()//to the screen!!!  
{  
	glDisable(GL_BLEND);	  
	  
	//drawCamera();  
	drawVideoTwo();  
	drawVideoMask();  
	  
	//drawVideo();//i don't need to draw this cause it is already in the mask  
	  
	//drawMotion();	  
	  
	  
}  
  
  
void testApp :: drawCamera()//actually i don't want to draw the camera  
{  
	ofSetColor( 0xFFFFFF );  
	//camera.draw( 0, 0, cameraWidth, cameraHeight );  
}  
  
  
  
  
  
void testApp :: drawMotion()//this is the motion, which I do want to draw but I cannot figure out how not to draw the background  
{  
	  
//	ofEnableAlphaBlending();//allows for image transparency  
//	ofSetColor( 255,0,0,50 );//changes color and transparency of motion  
	  
	//cameraMotionFloatImage.draw(0, 0, cameraWidth, cameraHeight );//maybe this is better?  
//	cameraMotionFloatImage.draw(360,280);//was 400 200  
//	ofDisableAlphaBlending();//stop allowing for image transparency  
	  
	  
}  
  
void testApp :: drawMotionDebug()  
{  
	//  
}  
  
  
//void testApp :: drawVideo()//this is the video with sun)in blobs and motion but i dont' need to draw it cause its already in the mask  
//{  
//	ofEnableAlphaBlending();//new  
//	ofSetColor( 255,255,255,50 );  
//	video.draw( 360, 280, 0, 0 );  
	//ofDisableAlphaBlending();//new  
//}  
  
void testApp :: drawVideoTwo()//this is the video w/out sun(background)  
{  
	ofEnableAlphaBlending();  
	ofSetColor(255,255,255,255);  
	videotwo.draw( 360, 280);  
	ofDisableAlphaBlending();  
}  
void testApp :: drawVideoMask()  
{  
	ofEnableAlphaBlending();  
	ofSetColor(255,255,255,255);//new		   
	maskTexture.draw( 360, 280 );  
	ofDisableAlphaBlending();  
}  
  
  
	  
}  
  
  

  
#include "testApp.h"  
  
  
//--------------------------------------------------------------  
void testApp::setup(){  
	  
	ofSetFrameRate(60);  
	threshold = 60;  
	bLearnBackground = true;  
	vidGrabber.initGrabber(320,240);  
	colorImg.allocate(320,240);  
	grayImg.allocate(320,240);  
	bgImg.allocate(320,240);  
	  
	video.loadMovie( "watersunreflect.mov" );  
	video.setLoopState(OF_LOOP_NORMAL);  
	video.play();  
	  
	videotwo.loadMovie( "watersun640_480.mov" );  
	videotwo.setLoopState(OF_LOOP_NORMAL);  
	videotwo.play();  
	  
	}  
  
//--------------------------------------------------------------  
void testApp::update(){  
	  
	  
	  
	ofBackground (100,100,100);  
	vidGrabber.grabFrame();  
	if(vidGrabber.isFrameNew() ) {  
		colorImg = vidGrabber.getPixels();  
		  
		grayImg = colorImg;  
		if (bLearnBackground) {  
			bgImg = grayImg;  
			bLearnBackground = false;  
		}  
		  
		grayImg.absDiff( bgImg);  
		grayImg.blur(11);  
		grayImg.threshold (threshold);  
		  
		contourFinder.findContours(grayImg, 50, 20000, 10, false, true);  
		blobTracker.trackBlobs (contourFinder.blobs) ;  
		}  
	  
}  
  
  
//--------------------------------------------------------------  
void testApp::draw(){  
	  
	ofEnableAlphaBlending();  
	  
	ofSetColor( 0xffffff);  
	//colorImg.draw(20,200);  
	//grayImg.draw (360,200);  
	  
	videotwo.draw(0,0);  
	blobTracker.draw( 320, 240);  
	  
	ofDisableAlphaBlending();  
  
	  
}  
  
  
  

This is working a little bit but I could still really use some suggestions if anyone can help. Now I can draw the shape I’m looking for with the following code but I still have no idea how to incorporate this into the code I posted in the previous post to make it work with the motion mask? And fill the shape with the second video instead of a color? By adding it to the mask? I like how the mask looks so much better than this (softer) so I want to try to add them together…Is that a good idea? or maybe not?

  
void ofxCvBlobTracker::draw( float x, float y ) {  
	ofEnableAlphaBlending();  
	ofSetColor( 255,255,255,50 );  
    glPushMatrix();  
    glTranslatef( x, y, 2.0 );  
  
	ofFill();  
	for( int i=0; i<blobs.size(); i++ ) {  
		ofBeginShape();  
		for( int j=0; j<blobs[i].pts.size(); j++ ){  
			ofVertex(blobs[i].pts[j].x, blobs[i].pts[j].y);  
			  
			  
			  
		//	blobs[i].pts[j].x, blobs[i].pts[j].y);  
		}  
	 ofEndShape();//me  
	}  
	   
  
	//ofSetColor(0xffffff);  
	for( int i=0; i<blobs.size(); i++ ) {  
		  
  
		glBegin(GL_TRIANGLE_FAN);  
		for( int j=0; j<blobs[i].pts.size(); j++ ) {  
						  
			glVertex2f( blobs[i].pts[j].x, blobs[i].pts[j].y );  
			glVertex2f(300, 260);//the point i want blobs to draw the shape to  
			  
		}  
		glEnd();  
		ofDisableAlphaBlending();  
		  
	}  
	  
    ofSetColor( 0xffffff );  
    for( int i=0; i<blobs.size(); i++ ) {  
        ostringstream docstring;  
        //docstring << blobs[i].id << endl;  
        docstring << findOrder(blobs[i].id) << endl;  
        ofDrawBitmapString( docstring.str(),  
                            blobs[i].centroid.x, blobs[i].centroid.y );  
    }  
	glPopMatrix();  
}  
  

Also, a smaller question- If I change this I can get the shape to be drawn on only the top half of the screen, how do I get it to draw on only the bottom half of the screen instead?

glBegin(GL_TRIANGLE_FAN);
for( int j=0; j<blobs[i].pts.size(); j++ ) {

glVertex2f( blobs[i].pts[j].x, blobs[i].pts[j].y/2 );
glVertex2f(300, 260);//the point i want blobs to draw the shape to

}
glEnd();

Thanks very much if you can help, appreciate it.

Hey mbvcloud

I actually looked through your example yesterday while trying to make mine work
http://forum.openframeworks.cc/t/can’t-run-ofx3dutils-example-on-mac/6655/0

A pure openCv method might work a bit smoother

If you dig around in the openCv addon in openFrameworks, namely ofxCvImage.h, you will see that they have a function “drawBlobIntoMe”, if you jump to its definition, you’ll see that it uses cvFillPoly with the CvPoints in a ofxCvBlob

  
  
	ofxCvGrayscaleImage cvMask; //Needs to be a single channel image for use as a mask later on  
	cvMask.allocate(width, height);   
	cvMask.drawBlobIntoMe(myBlob,255);  
  
	//once you have your mask, you can simply use the cvAdd function  
	//you will need "pointers" to the iplImages inside the ofxCvImages  
	IplImage* iplSource1; //the * means that iplSource is a pointer to an IplImage, not an IplImage its self  
	IplImage* iplSource2;  
	IplImage* iplDest;  
	IplImage* iplMask;  
	iplSource1	= cvSource1.getCvImage;  //Your first video  
	iplSource2	= cvSource2.getCvImage;  //Your second video  
	iplDest		= cvDest.getCvImage;  //Your destination image  
	iplMask		= cvMask.getCvImage;  //Your new mask image  
	  
	cvAdd(iplSource1, iplSource2, iplDest, iplMask);  
	//Then before you can draw the image, you need to tell cvDest that you have updated its cvImage  
	cvDest.flagImageChanged();  
	//Oh, and if your continually updating the mask, remember to clear() it before you update it again :)  
  

Haven’t tested it, but it should work in theory if I understand what you want to do correctly
Im new to this, so forgive me if it fails :slight_smile:
If your working with computer vision, I really recommend you check out this book http://amzn.to/qhefsw

Theron

Hi Theron, I want to thank you very much for taking time to look at and comment on my post. I have been trying to implement what you suggested all day but unfortunately, I am still totally confused and can’t quite get my project to work. I can’t seem to get the draw blob into me function to work and I have been trying to find code examples using this online but can’t find anything really. This is my first try at any programming so I have quite a bit to learn I guess.

Now I CAN:
1.capture the blobs and draw them in the triangle shape that I want(but just in white color not in pixels from video#2)
2.capture the motion and make it into a mask that shows the video #2

but I CAN’T
1.do both these things at the same time :frowning:

I think the problem is maybe that I am trying to take the absolute difference of the camera image two times, once to find the blobs and once to find the motion?? Because if I comment this part out in my program, say in the blob section, then the motion section works; and if I comment it out in the motion section, then the blob section works. I guess I’m just not sure what is the correct way to do it at the same time. I will definitely find the book you recommended, thank you. Here is what I have now, each part will work on its own but not together… I am sure there is a much, much better way to do this. Hopefully that book will help me or if you or anyone else has time to give suggestions I would really appreciate it.

Thank you very much

  
#include "testApp.h"  
  
  
//--------------------------------------------------------------  
void testApp::setup(){  
	  
  
  
	threshold = 60;  
	bLearnBackground = true;  
	vidGrabber.initGrabber(cameraWidth,cameraHeight);//i can't have this twice i think? (if I turn it off I can draw the mask but not blobs)  
	colorImg.allocate(cameraWidth,cameraHeight);  
	grayImg.allocate(cameraWidth,cameraHeight);  
	bgImg.allocate(cameraWidth,cameraHeight);  
	  
  
	  
	  
	  
	ofSetVerticalSync( true ); //i don't know what this is  
	  
	ofBackground( 255, 255,255 );//background color  
	  
	initVideoTwo();//second video  
	initCamera();//camera feed  
	initVideoMask();//video mask  
	initVideo();//first video  
	initMotion();//motion  
	  
	  
	  
	  
}  
  
  
  
  
void testApp :: initCamera()//this is catching the camera  
{  
	cameraWidth		= 640;  
	cameraHeight	= 480;  
	  
	camera.setVerbose( true );  
	camera.initGrabber( cameraWidth, cameraHeight );  
}  
  
void testApp :: initVideo()//this is the first video  
{  
	video.loadMovie( "watersunreflect.mov" );  
	video.play();  
}  
  
void testApp :: initVideoTwo()//this is the second video  
{  
	videotwo.loadMovie( "watersun640_480.mov" );  
	videotwo.play();  
	  
}  
  
  
void testApp :: initMotion()//this is the motion  
{  
	cameraColorImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayPrevImage.allocate( cameraWidth, cameraHeight );  
	cameraGrayDiffImage.allocate( cameraWidth, cameraHeight );  
	cameraDiffFloatImage.allocate( cameraWidth, cameraHeight );  
	cameraMotionFloatImage.allocate( cameraWidth, cameraHeight );  
	  
	cameraMotionFadeAmount = .95f;//allows for slow fade of motion drawn  
	  
	  
}  
  
void testApp :: initVideoMask()//this is the mask  
{  
	maskTexture.allocate( cameraWidth, cameraHeight, GL_RGBA );  
	  
	maskPixles = new unsigned char[ cameraWidth * cameraHeight * 4 ];  
}  
  
  
	  
//--------------------------------------------------------------  
void testApp::update(){  
	  
	  
	  
	ofBackground (100,100,100);  
	vidGrabber.grabFrame();  
	if(vidGrabber.isFrameNew() ) {  
		colorImg = vidGrabber.getPixels();  
	  
		grayImg = colorImg;  
		if (bLearnBackground) {  
			bgImg = grayImg;  
			bLearnBackground = false;  
		}  
		  
		grayImg.absDiff( bgImg);  
		grayImg.blur(11);  
		grayImg.threshold (threshold);  
		  
		contourFinder.findContours(grayImg, 50, 20000, 10, false, true);  
		blobTracker.trackBlobs (contourFinder.blobs) ;  
		  
		}  
	  
	  
	updateCamera();  
	  
	if( cameraNewFrame )  
	{  
		updateMotion( camera.getPixels() );  
		updateVideoMask();  
	}  
	  
	  
	  
	  
}  
  
void testApp :: updateCamera() //get the new camera frame  
{  
	camera.grabFrame();  
	cameraNewFrame = camera.isFrameNew();  
}  
  
void testApp :: updateMotion( unsigned char *pixels )//catch the new motion  
{  
	cameraColorImage.setFromPixels( pixels, cameraWidth, cameraHeight );  
	  
	cameraGrayPrevImage	= cameraGrayImage;  
	cameraGrayImage		= cameraColorImage;  
	  
	  
	cameraGrayDiffImage.absDiff( cameraGrayImage, cameraGrayPrevImage );  
	cameraGrayDiffImage.threshold( 30 );  
	  
	cameraDiffFloatImage	= cameraGrayDiffImage;  
	cameraMotionFloatImage	*= cameraMotionFadeAmount;  
	cameraMotionFloatImage	+= cameraDiffFloatImage;//I think this is where I need the 'draw blob into me?"  
	cameraMotionFloatImage.blur( 20 );  
	  
	  
}  
  
void testApp :: updateVideoMask()    
{  
	videoPixels		= video.getPixels();//whatever is in here is what fills the mask  
	motionPixles	= cameraMotionFloatImage.getPixels();  
	  
	for( int i=0; i<cameraWidth; i++ )  
	{  
		for( int j=0; j<cameraHeight; j++ )  
		{  
			int p = j * cameraWidth + i;  
			maskPixles[ p * 4 + 0 ] = videoPixels[ p * 3 + 0 ];   
			maskPixles[ p * 4 + 1 ] = videoPixels[ p * 3 + 1 ];   
			maskPixles[ p * 4 + 2 ] = videoPixels[ p * 3 + 2 ];   
			maskPixles[ p * 4 + 3 ] = motionPixles[ p ];  
		}  
	}  
	  
	maskTexture.loadData( maskPixles, cameraWidth, cameraHeight, GL_RGBA );  
}  
  
  
	  
  
  
  
//--------------------------------------------------------------  
void testApp::draw(){  
	  
	ofEnableAlphaBlending();  
	  
	ofSetColor( 0xffffff);  
	//colorImg.draw(20,200);  
	//grayImg.draw (360,200);  
	  
	videotwo.draw(0,0);//this is the background video  
	blobTracker.draw( 0, 0);//I want to draw this with the mask  
	  
	//drawCamera();  
	//drawVideoTwo();  
	drawVideoMask(); //I want to draw this  
	  
	//drawVideo();//i don't need to draw this because it is already in the mask  
	  
	//drawMotion();	  
	ofDisableAlphaBlending();  
	  
}  
  
  
void testApp :: drawCamera()//actually i don't want to draw the camera  
{  
	ofSetColor( 0xFFFFFF );  
	//camera.draw( 0, 0, cameraWidth, cameraHeight );  
}  
  
  
  
  
  
void testApp :: drawMotion()//this is the motion, which I do want to draw but I cannot figure out how to make the background transparent  
{  
	  
		ofEnableAlphaBlending();  
		ofSetColor( 255,255,255,255 );  
		cameraMotionFloatImage.draw(0, 0, cameraWidth, cameraHeight );  
		ofDisableAlphaBlending();  
	  
	  
}  
  
void testApp :: drawMotionDebug()  
{  
	//  
}  
  
  
//void testApp :: drawVideo()//this is the video with sun)that I want in blobs and motion but i dont' need to draw it cause its already in the mask  
//{  
//	ofEnableAlphaBlending();//new  
//	ofSetColor( 255,255,255,50 );  
//	video.draw( 360, 280, 0, 0 );  
//ofDisableAlphaBlending();//new  
//}  
  
void testApp :: drawVideoTwo()//this is the video w/out sun(background)  
{  
	ofEnableAlphaBlending();  
	ofSetColor(255,255,255,255);  
	videotwo.draw( 0, 0);  
	ofDisableAlphaBlending();  
}  
void testApp :: drawVideoMask()  
{  
	ofEnableAlphaBlending();  
	ofSetColor(255,255,255,255);	   
	maskTexture.draw( 0, 0 );  
	ofDisableAlphaBlending();  
}  
  
  
/code]  
  
  

Hi, I still can’t get this. Can anyone point me in the right direction of where I can learn about masks? or how to use the draw blob into function correctly? I feel like I’ve tried everything i know how to do :frowning: but I still can’t fill the blob shape with video pixels from the second video. do i even need a mask? can i just say that i want to fill the blob shape with pixels from the second video in here somewhere?

  
void ofxCvBlobTracker::draw( float x, float y ) {  
	  
   glPushMatrix();  
 glTranslatef( x, y, 2.0 );  
  
	  
	ofEnableAlphaBlending();  
	ofEnableSmoothing();  
	  
  
	for( int i=0; i<blobs.size(); i++ ) {  
		  
  
		glBegin(GL_LINES_ADJACENCY_EXT);  
		for( int j=0; j<blobs[i].pts.size(); j++ ) {  
			  
			  
						  
			glVertex2f( blobs[i].pts[j].x, blobs[i].pts[j].y );  
			glVertex2f(300, 260);//the point i want included in all blobs shapes  
			  
			  
			}  
		glEnd();  
		ofDisableAlphaBlending();  
		ofDisableSmoothing();  
		  
	}  
	  
    
	glPopMatrix();  
}