Unique Blob Identifiers using Kinect

Hi All,

I am using a kinect (model 1414) for a project and have a question regarding blob detection. I understand how to detect blobs through ofxContourFinder but am wondering how to identify each blob individually? Eventually I want to assign each blob a unique color.

I think this should be a relatively easy thing to do, but I am having trouble wrapping my head around how to do this.

Here’s the code if it helps. Line 167 is specifically where I’m struggling.

Thanks!

ofApp.h:

#pragma once

#include "ofMain.h"
#include "ofxKinect.h"
//#include "ofxGui.h"
#include "ofxOpenCv.h"
#include "ofxCv.h"

using namespace cv;
using namespace ofxCv;

class ofApp : public ofBaseApp{

	public:
		void setup();
		void update();
		void draw();

		void keyPressed(int key);
		void keyReleased(int key);
		void mouseMoved(int x, int y );
		void mouseDragged(int x, int y, int button);
		void mousePressed(int x, int y, int button);
		void mouseReleased(int x, int y, int button);
		void mouseEntered(int x, int y);
		void mouseExited(int x, int y);
		void windowResized(int w, int h);
		void dragEvent(ofDragInfo dragInfo);
		void gotMessage(ofMessage msg);
    
    
    ofxKinect kinect;
    ofMesh mesh;
    
    ofxCvContourFinder contourFinder;
    ofxCvGrayscaleImage grayImage;
    
    int grayFarThresh, grayNearThresh;
    
    ofEasyCam cam;
    
    int angle;
    
    float randomColor;
    
    float nearThresh, farThresh;
    
    const int skip = 4;
		
};

ofApp.cpp:

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){
    ofSetLogLevel(OF_LOG_VERBOSE);
    ofBackground(0);
    
//    ofSetFrameRate(30);

    kinect.setRegistration(true);
    kinect.init();
    kinect.open();
    
    angle = 20;
    kinect.setCameraTiltAngle(angle);
    
    grayImage.allocate(kinect.width, kinect.height);
    grayFarThresh = 145;
    grayNearThresh = 255;

    
    farThresh = -800;
    nearThresh = 1000;
    
//    randomColor = ofRandom(3);
//    cout << randomColor << endl;
    
    
}

//--------------------------------------------------------------
void ofApp::update(){
    mesh.setMode(OF_PRIMITIVE_TRIANGLES);
    kinect.update();
    

    if(kinect.isFrameNew()){
        
        mesh.clear();
        
        grayImage.setFromPixels(kinect.getDepthPixels());
        
        ofPixels & pix = grayImage.getPixels();
        int numPixels = pix.size();
        for(int i = 0; i < numPixels; i++) {
            if(pix[i] < grayNearThresh && pix[i] > grayFarThresh) {
                pix[i] = 255;
            } else {
                pix[i] = 0;
            }
        }
        grayImage.flagImageChanged();
        contourFinder.findContours(grayImage, 50, (kinect.width*kinect.height)/2, 20, false);
        cout << "number of blobs: " << contourFinder.nBlobs << endl;
        
        
    int width = kinect.width/skip;
    int height = kinect.height/skip;
    for(int y=0; y<height; y++){
        for(int x=0; x<width; x++){
            ofVec3f vertex = kinect.getWorldCoordinateAt(x * skip , y * skip);
            ofColor kinCol = kinect.getColorAt(x * skip, y * skip);
            ofVec3f normal = ofVec3f(0,0,0);
            
//            if(vertex.z == 0){
//                kinCol.a = 0;
//            }
            
            // Set depth offset
            vertex.z *= -1;
            vertex.z += 1000;
            
            
            if(vertex.z > nearThresh || vertex.z <farThresh){
                kinCol.a = 0;
            }
      
                mesh.addVertex(vertex);
                mesh.addNormal(normal);
                mesh.addColor(kinCol);
   
    }

}
        for(int y = 0; y<height-1; y++){
            for(int x = 0; x<width-1; x++){
        
            mesh.addIndex(x+y*width);         // 0
            mesh.addIndex((x+1)+y*width);     // 1
            mesh.addIndex(x+(y+1)*width);     // 10
            
            mesh.addIndex((x+1)+y*width);     // 1
            mesh.addIndex((x+1)+(y+1)*width); // 11
            mesh.addIndex(x+(y+1)*width);     // 10
        }
    }
        
    auto& ind = mesh.getIndices();
    auto &verts = mesh.getVertices();
    auto &norms = mesh.getNormals();
    
    for(int i=0; i<ind.size(); i+=3){
         int ia = ind[i];
         int ib = ind[i+1];
         int ic = ind[i+2];
        
        ofVec3f e1 = verts[ia] - verts[ib];
        ofVec3f e2 = verts[ic] - verts[ib];
        
        ofVec3f normal = e2.cross(e1);
        
        norms[ia] += normal;
        norms[ib] += normal;
        norms[ic] += normal;

    }
        for(int i=0; i<norms.size(); i++){
            norms[i] = glm::normalize(norms[i]);
        }
}
}

//--------------------------------------------------------------
void ofApp::draw(){
    
//    grayImage.draw(10, 320, 400, 300);
    contourFinder.draw(10, 320, 400, 300);
    
    
    ofPushMatrix();
    cam.begin();
    ofEnableDepthTest();
    ofTranslate(0, 0, -1000);
    ofScale(-1,-1, 1);
    ofPushStyle();
    
    
    vector<ofMeshFace> faces = mesh.getUniqueFaces();
    for(int i =0; i<faces.size(); i++){
        
        ofVec3f v1 = faces[i].getVertex(0);
        ofVec3f v2 = faces[i].getVertex(1);
        ofVec3f v3 = faces[i].getVertex(2);
        
        ofVec3f n1 = faces[i].getNormal(0);
        ofVec3f n2 = faces[i].getNormal(1);
        ofVec3f n3 = faces[i].getNormal(2);
        
        ofFloatColor c1 = faces[i].getColor(0);
        ofFloatColor c2 = faces[i].getColor(1);
        ofFloatColor c3 = faces[i].getColor(2);
        
        if(c1.a > 0 && c2.a > 0 && c3.a > 0) {
            
            //dark brown
            ofColor furBrown = ofColor(78, 54, 41);
        
            //forest green
            ofColor furGreen = ofColor(33, 68, 2);
            
            //fur white
            ofColor white = ofColor(255,255,255);
            
            //mesh triangles
//            ofDrawTriangle(v1, v2, v3);
   
            for(int i=0; i< contourFinder.nBlobs; i++){
                ofSetColor(ofRandom(255), ofRandom(255) ,0);
            }
            
        ofVec3f startHair = v1 *1.75;
        ofVec3f endHair = ofVec3f(v1 *1.75 + n1*1.75);
        for (int j = 0; j< 15; j++){
//            ofSetLineWidth(2);
            ofDrawLine(startHair, endHair);
            startHair = endHair;
            endHair += ofVec3f(0, ofRandom(j), 0) + n1*ofRandom(j);
        }
        }
        }
    
    ofPopStyle();
    cam.end();
    ofDisableDepthTest();
    ofPopMatrix();
}

//--------------------------------------------------------------
void ofApp::keyPressed(int key){
    switch(key){
        case OF_KEY_UP:
            angle++;
            if(angle>30) angle=30;
            kinect.setCameraTiltAngle(angle);
            break;
            
        case OF_KEY_DOWN:
            angle--;
            if(angle<-30) angle=-30;
            kinect.setCameraTiltAngle(angle);
            break;
    }
    
    if(key == 'f'){
        ofToggleFullscreen();
    }

}

//--------------------------------------------------------------
void ofApp::keyReleased(int key){

}

//--------------------------------------------------------------
void ofApp::mouseMoved(int x, int y ){

}

//--------------------------------------------------------------
void ofApp::mouseDragged(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mousePressed(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mouseReleased(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mouseEntered(int x, int y){

}

//--------------------------------------------------------------
void ofApp::mouseExited(int x, int y){

}

//--------------------------------------------------------------
void ofApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void ofApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void ofApp::dragEvent(ofDragInfo dragInfo){ 

}

I am just guessing, but maybe you can identify your blobs by keeping track of the last N positions of each blob?

if you are using the ofxCv ContourFinder you can use the getLabel( int index ) method to get the unique index of the blob, for example in your code you could use a std::vector<ofColor> colors to use different color based on the label:

     for(int i=0; i< contourFinder.nBlobs; i++){
         int id = contourFinder.getLabel(i);
         int c = id % colors.size();
         ofSetColor( colors[c] );
     }

to get the label of the i blob. The contour finder has two method to set the max distance before a blob is considered a new blob and the persistance of the id:

    // set contour tracker parameters
    contourFinder.getTracker().setPersistence(persistence);
    contourFinder.getTracker().setMaximumDistance(maxDistance);

set it to values good for your working conditions otherwise the blob labels could change a lot, stopping being useful.

1 Like

Hey npisanti, thanks this is really helpful! However, when I try to implement the countourFinder.getLabel(i) I get the following message: No member named ‘getLabel’ in ‘ofxCvContourFinder’

This is strange because I know that it the getLabel function is part of the ContourFinder class. Any idea how to get around this?

Thanks,
Noah

ofxCvContourFinder is the ofxOpenCv contour finder, you should use ofxCv::ContourFinder to have that method

Ah ok thanks! When I make that shift in my header, I start getting a bunch of errors around .findCountour and .nblobs. Do you know if there’s a way to just make a reference, or do I have to transpose all the snytax to work with ofxCv::ContourFinder?

@npisantu is referring to another addon, https://github.com/kylemcdonald/ofxCv

Ah got it - thanks! I thought that’s what I was using as well! Just looked through the ofxCv documentation.

Hey sorry - one last question! I have the colors changing based on int id = contourFinder.getLabel§; but that is changing all the colors on the screen. How should I go about changing the color of each blob individually?

Here’s the code I have for the color changing for now:

            for(int p=0; p< contourFinder.size(); p++){
                int id = contourFinder.getLabel(p);
                cout << p << endl;
                int c = id % colors.size();
                if(id ==1){
                    ofSetColor(furBrown);
                }
                else if(id == 2){
                    ofSetColor(furGreen);
                }
                else if(id == 3){
                    ofSetColor(white);
                }
                else{
                    ofSetColor( colors[c] );
                }
            }

in your code id will be equal to 1 or 2 or 3 just for the first acquired blobs, then to next blobs other label will be assigned (incrementally), never returning to 1, 2 or 3 again (and this could happen really fast). So depending on your input conditions it could be really hard to do stable operation on selected blobs.

to add to this, you can end up very easily with occlusion issues, which then needs quite a lot more design and code consideration.
(blob 2 disappears as it goes behind blob 1 and then reappears the other side as blob 3)