Masking with OpenCV functions

Hello,

After wrestling with this problem for quite a while, I thought I’d share a solution for masking an OpenCV Image with a hand-drawn Bezier-shaped mask. This is super useful for all kinds of tracking scenarios including when using the Kinect.

My code is based on this post, which clearly shows the OpenCV elements needed to achieve masking using cvCopy(): http://forum.openframeworks.cc/t/working-with-opencv-functions/6652/0

My solution is leaner CPU-wise because it doesn’t copy the image but applies the b/w mask directly using cvSet(). In order to achieve that I had to find out how to go from an drawn mask captured in an FBO, through ofPixels to an ofImage to finally reach the OpenCV IplImage containing the mask.

Using the cvSet() function has the added advantage of permitting the color of the masked parts to be set, quite useful to attenuate the sharp edges of the mask for adaptive thresholding algorithms for example.

Usage:
Hit the b key to go into the mask-editing mode.
Hit uppercase M to reset the mask.
Move the clicked mouse in the y position to change the mask-color (when not in editing mode).

enjoy

jasch


testApp.h

  
  
#ifndef _TEST_APP  
#define _TEST_APP  
  
#include "ofMain.h"  
#include "ofxOpenCv.h"  
#include "ofxXmlSettings.h"  
  
#define NUM_MASK_VERTICES 12  
  
typedef struct {  
	float 	x;  
	float 	y;  
	bool 	bBeingDragged;  
	bool 	bOver;  
	float 	radius;  
}draggableVertex;  
  
class testApp : public ofBaseApp  
{      
    void setup();  
    void update();  
    void draw();  
      
    void mouseMoved(int x, int y );  
    void mouseDragged(int x, int y, int button);  
    void mousePressed(int x, int y, int button);  
    void mouseReleased();  
      
    void keyPressed  (int key);  
      
    void applyMask();  
    void editMaskDrawing(int x, int y);  
    void saveMask();  
    void loadMask();      
    void resetMask();  
  
    ofVideoGrabber vidGrabber;  
    ofxXmlSettings maskXML;  
  
    int width, height;  
    bool editMask;  
    bool maskInited;  
    bool maskApplied;  
      
    ofxCvColorImage cvImage;  
    ofxCvColorImage cvMask;  
    ofxCvGrayscaleImage cvGrayMask;    
      
    IplImage * iplImg;    
    IplImage * iplMask;    
    IplImage * iplGrayMask;    
     
    ofFbo maskFBO;  
    ofImage maskImage;  
    ofPixels maskPixels;  
    CvScalar maskFill;  
  
    draggableVertex maskVertices[NUM_MASK_VERTICES];  
};  
  
#endif  
  

testApp.cpp

  
  
#include "testApp.h"    
  
void testApp::setup()   
{      
    ofSetFrameRate(60);  
    width   = 640;    
    height  = 480;    
      
    cvImage.allocate(width,height);    
    cvMask.allocate(width,height);    
    cvGrayMask.allocate(width,height);  
      
    iplImg = cvImage.getCvImage(); // the input image    
    iplMask = cvMask.getCvImage(); // the color masking image   
    iplGrayMask = cvGrayMask.getCvImage(); // the grayscale mask image  
      
    maskPixels.allocate(width, height, OF_PIXELS_RGB);    // the channel count for all three have to be indentical!  
    maskImage.allocate(width, height, OF_IMAGE_COLOR);  
    maskFBO.allocate(width, height, GL_RGB);    // fbo for drawing a new mask  
      
    maskFill = cvScalarAll(36); // the color of the masked part  
      
    editMask = false;  
    maskInited = false;  
    maskApplied = false;  
      
    loadMask();  
      
    vidGrabber.initGrabber(width, height);  
}    
  
void testApp::update()   
{  
    vidGrabber.update();       
  
    if (vidGrabber.isFrameNew()) {  
        cvImage.setFromPixels(vidGrabber.getPixels(), width, height);  
          
        if(maskApplied == false)  
            applyMask();  
          
        // this is where the mask is calculated, directly in the IplImages within the cvImages  
        cvSet(iplImg, maskFill, iplGrayMask);   
    }    
}    
  
void testApp::draw()   
{  
    ofSetColor(255, 255, 255, 255);  
    cvImage.draw(5,5);  
  
    if(editMask || (maskInited == false) ) {  
        editMaskDrawing(5, 5);  
    }  
}   
  
#pragma mark -  
#pragma mark masking functions  
  
void testApp::applyMask()  
{  
    cvMask.setFromPixels(maskImage.getPixels(), width, height); // copy it to a cvColorImage  
    cvInRangeS(iplMask, cvScalarAll(250), cvScalarAll(255), iplGrayMask);  
    maskApplied = true;  
}  
  
void testApp::editMaskDrawing(int x, int y)  
{  
    maskFBO.begin();  
    ofDisableAlphaBlending();  
    ofFill();  
    ofSetColor(255, 255, 255);  
    ofRect(0, 0, 640, 480);  
    ofSetColor(0, 0, 0);  
    ofBeginShape();  
    ofVertex(maskVertices[0].x, maskVertices[0].y);      
    ofBezierVertex(maskVertices[1].x, maskVertices[1].y, maskVertices[2].x,maskVertices[2].y, maskVertices[3].x,maskVertices[3].y);  
    ofVertex(maskVertices[3].x, maskVertices[3].y);  
    ofBezierVertex(maskVertices[4].x, maskVertices[4].y, maskVertices[5].x,maskVertices[5].y, maskVertices[6].x,maskVertices[6].y);  
    ofVertex(maskVertices[6].x, maskVertices[6].y);  
    ofBezierVertex(maskVertices[7].x, maskVertices[7].y, maskVertices[8].x,maskVertices[8].y, maskVertices[9].x,maskVertices[9].y);  
    ofVertex(maskVertices[9].x, maskVertices[9].y);  
    ofBezierVertex(maskVertices[10].x, maskVertices[10].y, maskVertices[11].x,maskVertices[11].y, maskVertices[0].x,maskVertices[0].y);  
    ofEndShape();  
  
    maskFBO.end();  
    maskFBO.readToPixels(maskPixels);  
    maskImage.setFromPixels(maskPixels);  
      
    maskInited = true;  
    maskApplied = false;  
      
    ofEnableAlphaBlending();  
    ofSetColor(255, 255, 255, 63); // transparent, so that we can see what we're doing  
      
    // draws the bezier control points  
    ofBeginShape();  
    for (int i = 0; i < NUM_MASK_VERTICES; i++) {  
        ofVertex(maskVertices[i].x+5, maskVertices[i].y+5);  
        if (maskVertices[i].bOver == true) {  
            ofFill();  
        } else {  
            ofNoFill();  
        }  
        if(i % 3 == 0) {  
            ofSetColor(200,200,200, 255);  
            ofRect(maskVertices[i].x+1.5, maskVertices[i].y+1.5, 7, 7);  
        }else{  
            ofSetColor(255,255,255, 255);  
            ofCircle(maskVertices[i].x+5, maskVertices[i].y+5, 3.5);  
        }  
    }   
    ofVertex(maskVertices[0].x+5, maskVertices[0].y+5);  
    ofNoFill();  
    ofEndShape();  
}  
  
void testApp::saveMask()  
{  
    maskXML.setValue("mask:vertex_0:x", maskVertices[0].x);  
    maskXML.setValue("mask:vertex_0:y", maskVertices[0].y);  
    maskXML.setValue("mask:vertex_1:x", maskVertices[1].x);  
    maskXML.setValue("mask:vertex_1:y", maskVertices[1].y);  
    maskXML.setValue("mask:vertex_2:x", maskVertices[2].x);  
    maskXML.setValue("mask:vertex_2:y", maskVertices[2].y);  
    maskXML.setValue("mask:vertex_3:x", maskVertices[3].x);  
    maskXML.setValue("mask:vertex_3:y", maskVertices[3].y);  
    maskXML.setValue("mask:vertex_4:x", maskVertices[4].x);  
    maskXML.setValue("mask:vertex_4:y", maskVertices[4].y);  
    maskXML.setValue("mask:vertex_5:x", maskVertices[5].x);  
    maskXML.setValue("mask:vertex_5:y", maskVertices[5].y);  
    maskXML.setValue("mask:vertex_6:x", maskVertices[6].x);  
    maskXML.setValue("mask:vertex_6:y", maskVertices[6].y);  
    maskXML.setValue("mask:vertex_7:x", maskVertices[7].x);  
    maskXML.setValue("mask:vertex_7:y", maskVertices[7].y);  
    maskXML.setValue("mask:vertex_8:x", maskVertices[8].x);  
    maskXML.setValue("mask:vertex_8:y", maskVertices[8].y);  
    maskXML.setValue("mask:vertex_9:x", maskVertices[9].x);  
    maskXML.setValue("mask:vertex_9:y", maskVertices[9].y);  
    maskXML.setValue("mask:vertex_10:x", maskVertices[10].x);  
    maskXML.setValue("mask:vertex_10:y", maskVertices[10].y);  
    maskXML.setValue("mask:vertex_11:x", maskVertices[11].x);  
    maskXML.setValue("mask:vertex_11:y", maskVertices[11].y);  
      
    maskXML.saveFile("maskPoints.xml");  
}  
  
void testApp::loadMask()  
{  
    bool result = maskXML.loadFile("maskPoints.xml");  
	if(result) {  
        maskVertices[0].x = maskXML.getValue("mask:vertex_0:x", 0);  
        maskVertices[0].y = maskXML.getValue("mask:vertex_0:y", 0);  
        maskVertices[1].x = maskXML.getValue("mask:vertex_1:x", 213);  
        maskVertices[1].y = maskXML.getValue("mask:vertex_1:y", 0);  
        maskVertices[2].x = maskXML.getValue("mask:vertex_2:x", 426);  
        maskVertices[2].y = maskXML.getValue("mask:vertex_2:y", 0);  
        maskVertices[3].x = maskXML.getValue("mask:vertex_3:x", 640);  
        maskVertices[3].y = maskXML.getValue("mask:vertex_3:y", 0);  
        maskVertices[4].x = maskXML.getValue("mask:vertex_4:x", 640);  
        maskVertices[4].y = maskXML.getValue("mask:vertex_4:y", 160);  
        maskVertices[5].x = maskXML.getValue("mask:vertex_5:x", 640);  
        maskVertices[5].y = maskXML.getValue("mask:vertex_5:y", 320);  
        maskVertices[6].x = maskXML.getValue("mask:vertex_6:x", 640);  
        maskVertices[6].y = maskXML.getValue("mask:vertex_6:y", 480);  
        maskVertices[7].x = maskXML.getValue("mask:vertex_7:x", 426);  
        maskVertices[7].y = maskXML.getValue("mask:vertex_7:y", 480);  
        maskVertices[8].x = maskXML.getValue("mask:vertex_8:x", 213);  
        maskVertices[8].y = maskXML.getValue("mask:vertex_8:y", 480);  
        maskVertices[9].x = maskXML.getValue("mask:vertex_9:x", 0);  
        maskVertices[9].y = maskXML.getValue("mask:vertex_9:y", 480);  
        maskVertices[10].x = maskXML.getValue("mask:vertex_10:x", 0);  
        maskVertices[10].y = maskXML.getValue("mask:vertex_10:y", 320);  
        maskVertices[11].x = maskXML.getValue("mask:vertex_11:x", 0);  
        maskVertices[11].y = maskXML.getValue("mask:vertex_11:y", 160);  
        for(int i = 0; i < NUM_MASK_VERTICES; i++) {  
            maskVertices[i].bBeingDragged = 0;  
            maskVertices[i].bOver = 0;  
            maskVertices[i].radius = 10.0f;  
        }  
    }else{  
        resetMask();  
    }  
    maskInited = 0;  
    maskApplied = 0;  
}  
  
void testApp::resetMask()  
{  
    maskVertices[0].x = 0;  
    maskVertices[0].y = 0;  
    maskVertices[1].x = 213;  
    maskVertices[1].y = 0;  
    maskVertices[2].x = 426;  
    maskVertices[2].y = 0;  
    maskVertices[3].x = 640;  
    maskVertices[3].y = 0;  
    maskVertices[4].x = 640;  
    maskVertices[4].y = 160;  
    maskVertices[5].x = 640;  
    maskVertices[5].y = 320;  
    maskVertices[6].x = 640;  
    maskVertices[6].y = 480;  
    maskVertices[7].x = 426;  
    maskVertices[7].y = 480;  
    maskVertices[8].x = 213;  
    maskVertices[8].y = 480;  
    maskVertices[9].x = 0;  
    maskVertices[9].y = 480;  
    maskVertices[10].x = 0;  
    maskVertices[10].y = 320;  
    maskVertices[11].x = 0;  
    maskVertices[11].y = 160;  
    for(int i = 0; i < NUM_MASK_VERTICES; i++) {  
        maskVertices[i].bBeingDragged = 0;  
        maskVertices[i].bOver = 0;  
        maskVertices[i].radius = 10.0f;  
    }  
}  
  
#pragma mark -  
#pragma mark mouse functions  
  
void testApp::mouseMoved(int x, int y )  
{  
    if(editMask) {  
        for (int i = 0; i < NUM_MASK_VERTICES; i++){  
            float diffx = (x - 5) - maskVertices[i].x;  
            float diffy = (y - 5) - maskVertices[i].y;  
            float dist = sqrt(diffx*diffx + diffy*diffy);  
            if (dist < maskVertices[i].radius){  
                maskVertices[i].bOver = true;  
            } else {  
                maskVertices[i].bOver = false;  
            }	  
        }  
    }  
      
}  
  
void testApp::mouseDragged(int x, int y, int button){  
      
    if(editMask) {  
        for (int i = 0; i < NUM_MASK_VERTICES; i++){  
            if (maskVertices[i].bBeingDragged == true){  
                maskVertices[i].x = CLAMP((x - 5), 0, 650);  
                maskVertices[i].y = CLAMP((y - 5), 0, 490);  
            }  
        }  
    }else{  
        maskFill = cvScalarAll( CLAMP( ((y-5) * 0.533333333333333), 0, 255 ) );  
    }  
      
}  
  
void testApp::mousePressed(int x, int y, int button)  
{      
    if(editMask) {  
        for (int i = 0; i < NUM_MASK_VERTICES; i++){  
            float diffx = (x - 5) - maskVertices[i].x;  
            float diffy = (y - 5) - maskVertices[i].y;  
            float dist = sqrt(diffx*diffx + diffy*diffy);  
            if (dist < maskVertices[i].radius){  
                maskVertices[i].bBeingDragged = true;  
            } else {  
                maskVertices[i].bBeingDragged = false;  
            }	  
        }  
    }  
}  
  
void testApp::mouseReleased()  
{      
    bool saveMaskAtRelease = 0;  
    if(editMask) {  
        for (int i = 0; i < NUM_MASK_VERTICES; i++){  
            if( maskVertices[i].bBeingDragged == true) {  
                saveMaskAtRelease = 1;  
            }  
            maskVertices[i].bBeingDragged = false;	  
        }  
        if(saveMaskAtRelease) {  
            saveMask();  
            applyMask();  
        }  
    }  
}  
  
#pragma mark -  
#pragma mark keys  
  
void testApp::keyPressed  (int key)  
{  
    switch (key) {      
        case 'm':  
            saveMask();  
            break;  
        case 'M':  
            resetMask();  
            break;  
        case 'l':  
            applyMask();  
            break;  
        case 'b':  
            editMask = !editMask;  
            break;   
    }  
}  
  

I know this is an older post, but goddamn is this useful…might have to find a way to break this into a ofxCvMaskHelper class or something - thanks!