Unique Blob Identifiers using Kinect


#1

Hi All,

I am using a kinect (model 1414) for a project and have a question regarding blob detection. I understand how to detect blobs through ofxContourFinder but am wondering how to identify each blob individually? Eventually I want to assign each blob a unique color.

I think this should be a relatively easy thing to do, but I am having trouble wrapping my head around how to do this.

Here’s the code if it helps. Line 167 is specifically where I’m struggling.

Thanks!

ofApp.h:

#pragma once

#include "ofMain.h"
#include "ofxKinect.h"
//#include "ofxGui.h"
#include "ofxOpenCv.h"
#include "ofxCv.h"

using namespace cv;
using namespace ofxCv;

class ofApp : public ofBaseApp{

	public:
		void setup();
		void update();
		void draw();

		void keyPressed(int key);
		void keyReleased(int key);
		void mouseMoved(int x, int y );
		void mouseDragged(int x, int y, int button);
		void mousePressed(int x, int y, int button);
		void mouseReleased(int x, int y, int button);
		void mouseEntered(int x, int y);
		void mouseExited(int x, int y);
		void windowResized(int w, int h);
		void dragEvent(ofDragInfo dragInfo);
		void gotMessage(ofMessage msg);
    
    
    ofxKinect kinect;
    ofMesh mesh;
    
    ofxCvContourFinder contourFinder;
    ofxCvGrayscaleImage grayImage;
    
    int grayFarThresh, grayNearThresh;
    
    ofEasyCam cam;
    
    int angle;
    
    float randomColor;
    
    float nearThresh, farThresh;
    
    const int skip = 4;
		
};

ofApp.cpp:

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){
    ofSetLogLevel(OF_LOG_VERBOSE);
    ofBackground(0);
    
//    ofSetFrameRate(30);

    kinect.setRegistration(true);
    kinect.init();
    kinect.open();
    
    angle = 20;
    kinect.setCameraTiltAngle(angle);
    
    grayImage.allocate(kinect.width, kinect.height);
    grayFarThresh = 145;
    grayNearThresh = 255;

    
    farThresh = -800;
    nearThresh = 1000;
    
//    randomColor = ofRandom(3);
//    cout << randomColor << endl;
    
    
}

//--------------------------------------------------------------
void ofApp::update(){
    mesh.setMode(OF_PRIMITIVE_TRIANGLES);
    kinect.update();
    

    if(kinect.isFrameNew()){
        
        mesh.clear();
        
        grayImage.setFromPixels(kinect.getDepthPixels());
        
        ofPixels & pix = grayImage.getPixels();
        int numPixels = pix.size();
        for(int i = 0; i < numPixels; i++) {
            if(pix[i] < grayNearThresh && pix[i] > grayFarThresh) {
                pix[i] = 255;
            } else {
                pix[i] = 0;
            }
        }
        grayImage.flagImageChanged();
        contourFinder.findContours(grayImage, 50, (kinect.width*kinect.height)/2, 20, false);
        cout << "number of blobs: " << contourFinder.nBlobs << endl;
        
        
    int width = kinect.width/skip;
    int height = kinect.height/skip;
    for(int y=0; y<height; y++){
        for(int x=0; x<width; x++){
            ofVec3f vertex = kinect.getWorldCoordinateAt(x * skip , y * skip);
            ofColor kinCol = kinect.getColorAt(x * skip, y * skip);
            ofVec3f normal = ofVec3f(0,0,0);
            
//            if(vertex.z == 0){
//                kinCol.a = 0;
//            }
            
            // Set depth offset
            vertex.z *= -1;
            vertex.z += 1000;
            
            
            if(vertex.z > nearThresh || vertex.z <farThresh){
                kinCol.a = 0;
            }
      
                mesh.addVertex(vertex);
                mesh.addNormal(normal);
                mesh.addColor(kinCol);
   
    }

}
        for(int y = 0; y<height-1; y++){
            for(int x = 0; x<width-1; x++){
        
            mesh.addIndex(x+y*width);         // 0
            mesh.addIndex((x+1)+y*width);     // 1
            mesh.addIndex(x+(y+1)*width);     // 10
            
            mesh.addIndex((x+1)+y*width);     // 1
            mesh.addIndex((x+1)+(y+1)*width); // 11
            mesh.addIndex(x+(y+1)*width);     // 10
        }
    }
        
    auto& ind = mesh.getIndices();
    auto &verts = mesh.getVertices();
    auto &norms = mesh.getNormals();
    
    for(int i=0; i<ind.size(); i+=3){
         int ia = ind[i];
         int ib = ind[i+1];
         int ic = ind[i+2];
        
        ofVec3f e1 = verts[ia] - verts[ib];
        ofVec3f e2 = verts[ic] - verts[ib];
        
        ofVec3f normal = e2.cross(e1);
        
        norms[ia] += normal;
        norms[ib] += normal;
        norms[ic] += normal;

    }
        for(int i=0; i<norms.size(); i++){
            norms[i] = glm::normalize(norms[i]);
        }
}
}

//--------------------------------------------------------------
void ofApp::draw(){
    
//    grayImage.draw(10, 320, 400, 300);
    contourFinder.draw(10, 320, 400, 300);
    
    
    ofPushMatrix();
    cam.begin();
    ofEnableDepthTest();
    ofTranslate(0, 0, -1000);
    ofScale(-1,-1, 1);
    ofPushStyle();
    
    
    vector<ofMeshFace> faces = mesh.getUniqueFaces();
    for(int i =0; i<faces.size(); i++){
        
        ofVec3f v1 = faces[i].getVertex(0);
        ofVec3f v2 = faces[i].getVertex(1);
        ofVec3f v3 = faces[i].getVertex(2);
        
        ofVec3f n1 = faces[i].getNormal(0);
        ofVec3f n2 = faces[i].getNormal(1);
        ofVec3f n3 = faces[i].getNormal(2);
        
        ofFloatColor c1 = faces[i].getColor(0);
        ofFloatColor c2 = faces[i].getColor(1);
        ofFloatColor c3 = faces[i].getColor(2);
        
        if(c1.a > 0 && c2.a > 0 && c3.a > 0) {
            
            //dark brown
            ofColor furBrown = ofColor(78, 54, 41);
        
            //forest green
            ofColor furGreen = ofColor(33, 68, 2);
            
            //fur white
            ofColor white = ofColor(255,255,255);
            
            //mesh triangles
//            ofDrawTriangle(v1, v2, v3);
   
            for(int i=0; i< contourFinder.nBlobs; i++){
                ofSetColor(ofRandom(255), ofRandom(255) ,0);
            }
            
        ofVec3f startHair = v1 *1.75;
        ofVec3f endHair = ofVec3f(v1 *1.75 + n1*1.75);
        for (int j = 0; j< 15; j++){
//            ofSetLineWidth(2);
            ofDrawLine(startHair, endHair);
            startHair = endHair;
            endHair += ofVec3f(0, ofRandom(j), 0) + n1*ofRandom(j);
        }
        }
        }
    
    ofPopStyle();
    cam.end();
    ofDisableDepthTest();
    ofPopMatrix();
}

//--------------------------------------------------------------
void ofApp::keyPressed(int key){
    switch(key){
        case OF_KEY_UP:
            angle++;
            if(angle>30) angle=30;
            kinect.setCameraTiltAngle(angle);
            break;
            
        case OF_KEY_DOWN:
            angle--;
            if(angle<-30) angle=-30;
            kinect.setCameraTiltAngle(angle);
            break;
    }
    
    if(key == 'f'){
        ofToggleFullscreen();
    }

}

//--------------------------------------------------------------
void ofApp::keyReleased(int key){

}

//--------------------------------------------------------------
void ofApp::mouseMoved(int x, int y ){

}

//--------------------------------------------------------------
void ofApp::mouseDragged(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mousePressed(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mouseReleased(int x, int y, int button){

}

//--------------------------------------------------------------
void ofApp::mouseEntered(int x, int y){

}

//--------------------------------------------------------------
void ofApp::mouseExited(int x, int y){

}

//--------------------------------------------------------------
void ofApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void ofApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void ofApp::dragEvent(ofDragInfo dragInfo){ 

}

#2

I am just guessing, but maybe you can identify your blobs by keeping track of the last N positions of each blob?


#3

if you are using the ofxCv ContourFinder you can use the getLabel( int index ) method to get the unique index of the blob, for example in your code you could use a std::vector<ofColor> colors to use different color based on the label:

     for(int i=0; i< contourFinder.nBlobs; i++){
         int id = contourFinder.getLabel(i);
         int c = id % colors.size();
         ofSetColor( colors[c] );
     }

to get the label of the i blob. The contour finder has two method to set the max distance before a blob is considered a new blob and the persistance of the id:

    // set contour tracker parameters
    contourFinder.getTracker().setPersistence(persistence);
    contourFinder.getTracker().setMaximumDistance(maxDistance);

set it to values good for your working conditions otherwise the blob labels could change a lot, stopping being useful.


#4

Hey npisanti, thanks this is really helpful! However, when I try to implement the countourFinder.getLabel(i) I get the following message: No member named ‘getLabel’ in ‘ofxCvContourFinder’

This is strange because I know that it the getLabel function is part of the ContourFinder class. Any idea how to get around this?

Thanks,
Noah


#5

ofxCvContourFinder is the ofxOpenCv contour finder, you should use ofxCv::ContourFinder to have that method


#6

Ah ok thanks! When I make that shift in my header, I start getting a bunch of errors around .findCountour and .nblobs. Do you know if there’s a way to just make a reference, or do I have to transpose all the snytax to work with ofxCv::ContourFinder?


#7

@npisantu is referring to another addon, https://github.com/kylemcdonald/ofxCv


#8

Ah got it - thanks! I thought that’s what I was using as well! Just looked through the ofxCv documentation.


#9

Hey sorry - one last question! I have the colors changing based on int id = contourFinder.getLabel§; but that is changing all the colors on the screen. How should I go about changing the color of each blob individually?

Here’s the code I have for the color changing for now:

            for(int p=0; p< contourFinder.size(); p++){
                int id = contourFinder.getLabel(p);
                cout << p << endl;
                int c = id % colors.size();
                if(id ==1){
                    ofSetColor(furBrown);
                }
                else if(id == 2){
                    ofSetColor(furGreen);
                }
                else if(id == 3){
                    ofSetColor(white);
                }
                else{
                    ofSetColor( colors[c] );
                }
            }

#10

in your code id will be equal to 1 or 2 or 3 just for the first acquired blobs, then to next blobs other label will be assigned (incrementally), never returning to 1, 2 or 3 again (and this could happen really fast). So depending on your input conditions it could be really hard to do stable operation on selected blobs.


#11

to add to this, you can end up very easily with occlusion issues, which then needs quite a lot more design and code consideration.
(blob 2 disappears as it goes behind blob 1 and then reappears the other side as blob 3)