Can't seem to get Takashi's opticalflowBM method to work.

Hey, all, I’ve gotten Lucas-Kanade optical flow working but want to try a simple block method. I found Takashi and Joel’s method on another thread, and it all compiles fine, but I get the following error:

“OpenCV Error: Sizes of input arguments do not match () in unknown function, file
src\optflowbm.cpp, line 88”

Which I cannot track down anywhere.

  
//  
// ofxCvOpticalFlowBM.h - a OpenCV cvCalcOpticalFlowBM warpper for openFrameworks  
//  
// Copyright (C) 2008 Takashi Maekawa <takachin@generative.info>   
// Copyright (C) 2008 Satoru Higa  
//     All rights reserved.  
//     This is free software with ABSOLUTELY NO WARRANTY.  
//  
// You can redistribute it and/or modify it under the terms of   
// the GNU Lesser General Public License.  
//  
  
#pragma once  
#include <cv.h>  
#include "ofMain.h"  
#include "ofxCvConstants.h"  
#include "ofxCvGrayscaleImage.h"  
  
class ofxCvOpticalFlowBM  
{  
public:  
	ofxCvOpticalFlowBM(void);  
	~ofxCvOpticalFlowBM(void);  
		  
	void allocate(int _w, int _h);  
  
	void calc( ofxCvGrayscaleImage & pastImage,  
			   ofxCvGrayscaleImage & currentImage,					     
			   int size  
			  );  
  
	void setCalcStep(int _cols, int _rows);  
		  
	void reset();  
	void draw();  
  
//private:  
public:  
  
	static const int DEFAULT_CAPTURE_WIDTH = 320;  
	static const int DEFAULT_CAPTURE_HEIGHT = 240;  
  
	int captureWidth;  
	int captureHeight;  
  
	static const int DEFAULT_CAPTURE_COLS_STEP = 4;  
	static const int DEFAULT_CAPTURE_ROWS_STEP = 4;  
  
	int captureColsStep;  
	int captureRowsStep;  
  
    int block_size;  
    int shift_size;  
  
	int rows,cols;  
	//int cw, ch;  
  
	CvSize block;  
	CvSize shift;  
	CvSize max_range;  
  
	CvMat *vel_x, *vel_y;  
  
};  
  

and

  
//  
// ofxCvOpticalFlowBM.c - a OpenCV cvCalcOpticalFlowBM warpper for openFrameworks  
//  
// Copyright (C) 2008 Takashi Maekawa <takachin@generative.info>   
// Copyright (C) 2008 Satoru Higa  
//     All rights reserved.  
//     This is free software with ABSOLUTELY NO WARRANTY.  
//  
// You can redistribute it and/or modify it under the terms of   
// the GNU Lesser General Public License.  
//  
#include <stdio.h>  
#include "ofxCvOpticalFlowBM.h"  
  
ofxCvOpticalFlowBM::ofxCvOpticalFlowBM(void)  
{  
	captureWidth = DEFAULT_CAPTURE_WIDTH;  
	captureHeight = DEFAULT_CAPTURE_HEIGHT;  
}  
  
ofxCvOpticalFlowBM::~ofxCvOpticalFlowBM(void)  
{  
  // TODO : release cv matrices   
  //cvReleaseImage(&vel_x);  
  //cvReleaseImage(&vel_y);  
}  
	  
void ofxCvOpticalFlowBM::allocate(int _w, int _h){  
	captureWidth = _w;  
	captureHeight = _h;  
	  
    //cw = 320; ch = 240;  
  
    block_size = 10;  
    shift_size = 1;  
      
    rows = int(ceil(double(captureHeight) / block_size));  
    cols = int(ceil(double(captureWidth) / block_size));  
      
    vel_x = cvCreateMat (rows, cols, CV_32FC1);  
    vel_y = cvCreateMat (rows, cols, CV_32FC1);  
      
    cvSetZero(vel_x);  
    cvSetZero(vel_y);  
      
    block = cvSize (block_size, block_size);  
    shift = cvSize (shift_size, shift_size);  
    max_range = cvSize (10, 10);  
  
}  
  
void ofxCvOpticalFlowBM::setCalcStep(int _cols, int _rows){  
	captureColsStep = _cols;  
	captureRowsStep = _rows;  
}  
  
void ofxCvOpticalFlowBM::calc( ofxCvGrayscaleImage & pastImage,  
					   ofxCvGrayscaleImage & currentImage,					     
					   int size  
					   )  
{  
  
	cvCalcOpticalFlowBM(pastImage.getCvImage(), currentImage.getCvImage(),   
            block, shift, max_range, 0, vel_x, vel_y);  
}  
  
void ofxCvOpticalFlowBM::draw(void){  
  
	ofEnableAlphaBlending();  
	ofSetHexColor(0xffffff);  
	ofNoFill();  
  
	// draw for debug display  
	int x, y, dx, dy;  
    for(int y = 0; y < rows; y++){  
        for(int x = 0; x < cols; x++){  
            int dx = (int) cvGetReal2D (vel_x, y, x);  
            int dy = (int) cvGetReal2D (vel_y, y, x);  
            int xx = x * block_size;  
            int yy = y * block_size;  
            ofLine(xx, yy, xx + dx, yy + dy);  
        }  
    }  
	ofDisableAlphaBlending();  
}  
  

  
#pragma once  
  
#include "ofMain.h"  
  
#include "ofxOpenCv.h"  
  
//#define _USE_LIVE_VIDEO		// uncomment this to use a live camera  
								// otherwise, we'll use a movie file  
  
class testApp : public ofBaseApp{  
  
	public:  
		void setup();  
		void update();  
		void draw();  
		  
		void keyPressed(int key);  
		void keyReleased(int key);  
		void mouseMoved(int x, int y );  
		void mouseDragged(int x, int y, int button);  
		void mousePressed(int x, int y, int button);  
		void mouseReleased(int x, int y, int button);  
		void windowResized(int w, int h);  
		void dragEvent(ofDragInfo dragInfo);  
		void gotMessage(ofMessage msg);		  
  
        #ifdef _USE_LIVE_VIDEO  
		  ofVideoGrabber 		vidGrabber;  
		#else  
		  ofVideoPlayer 		vidPlayer;  
		#endif  
  
        ofxCvColorImage			colorImg;  
  
        ofxCvGrayscaleImage 	grayImage;  
		ofxCvGrayscaleImage 	grayBg;  
		ofxCvGrayscaleImage 	grayPast;  
  
		  
		int rows, cols;  
		int block_size;  
		int shift_size;  
		  
		  
		CvMat *velx, *vely;  
		CvSize block;  
		CvSize shift;  
		CvSize max_range;  
		  
		//ofxCvContourFinder 	contourFinder;  
		//ofxCvOpticalFlowLK	flow;  
		ofxCvOpticalFlowBM	flow;  
		  
  
		int 				threshold;  
		bool				bLearnBakground;  
		int					w,h;  
  
  
};  
  
  

and the .cpp

  
#include "testApp.h"  
  
//--------------------------------------------------------------  
void testApp::setup(){  
  
	#ifdef _USE_LIVE_VIDEO  
        vidGrabber.setVerbose(true);  
        vidGrabber.initGrabber(320,240);  
	#else  
        vidPlayer.loadMovie("fingers.mov");  
        vidPlayer.play();  
	#endif  
	w = vidPlayer.width;  
	h = vidPlayer.height;  
    colorImg.allocate(w,h);  
	grayImage.allocate(w,h);  
	grayBg.allocate(w,h);  
	grayPast.allocate(w,h);  
  
	bLearnBakground = true;  
	threshold = 75;  
	flow.allocate(w,h);  
  
}  
  
//--------------------------------------------------------------  
void testApp::update(){  
	ofBackground(100,100,100);  
  
    bool bNewFrame = false;  
  
	#ifdef _USE_LIVE_VIDEO  
       vidGrabber.grabFrame();  
	   bNewFrame = vidGrabber.isFrameNew();  
    #else  
        vidPlayer.idleMovie();  
        bNewFrame = vidPlayer.isFrameNew();  
	#endif  
  
	if (bNewFrame){  
		  
		#ifdef _USE_LIVE_VIDEO  
            colorImg.setFromPixels(vidGrabber.getPixels(), 320,240);  
	    #else  
			      
			colorImg.setFromPixels(vidPlayer.getPixels(), w,h);  
        #endif  
		  
        grayImage = colorImg;  
		if (bLearnBakground == true){  
			grayBg = grayImage;		// the = sign copys the pixels from grayImage into grayBg (operator overloading)  
			bLearnBakground = false;  
		}  
  
		// take the abs value of the difference between background and incoming and then threshold:  
		//grayDiff.absDiff(grayBg, grayImage);  
	  
		grayImage.blurGaussian(6);  
		grayImage.blurMedian(25);  
		  
		grayImage.threshold(threshold);  
		grayImage.blurGaussian(1);  
		//printf("%p %p\n", grayPast.getCvImage(), grayImage.getCvImage());  
		flow.calc(grayPast, grayImage,10);  
		grayPast = grayImage;  
	}  
  
}  
  
//--------------------------------------------------------------  
void testApp::draw(){  
  
	// draw the incoming, the grayscale, the bg and the thresholded difference  
	ofSetHexColor(0xFFFFFF);  
	colorImg.draw(360,20);  
	grayImage.draw(360,h+40);  
	//grayBg.draw(20,280);  
	//grayDiff.draw(360,20);  
  
	//ofFill();  
	//ofSetHexColor(0x333333);  
	//ofRect(360,20,320,240);  
	ofSetHexColor(0xffffff);  
	//glPushMatrix();  
		//glTranslatef(360,20,0);  
	flow.draw();  
	//glPopMatrix();  
  
	//float horizontalVelocityAtXY = cvGetReal2D( flow.vel_x, y, x );    
	//float verticalVelocityAtXY = cvGetReal2D( flow.vel_y, y, x );    
    /*//get the average vector back and scale  
    //we scale by a lot because the number of changed pixels  
    //is quite small in comparison to the total number  
    //if you wanted to get motion but didn't care about magnitude  
    //you could normalize this result.  
    ofVec2f overall = vision.getOverallMotionFromField();  
    overall *= 30000.0;  
  
    //our center point - just for drawing a line from the center of the screen  
    ofPoint center(ofGetWidth()/2, ofGetHeight()/2);  
  
    //draw a line representing the average vector  
    ofSetHexColor(0xFF00FF);  
    ofLine(center.x, center.y, center.x + overall.x, center.y + overall.y);  
  
    //draw just the horizontal component  
    ofSetHexColor(0x0000FF);  
    ofLine(center.x, center.y, center.x + overall.x, center.y);*/  
  
	//ofSetHexColor(0xffffff);  
  
	// then draw the contours:  
  
	/*ofFill();  
	ofSetHexColor(0x333333);  
	ofRect(360,540,320,240);  
	ofSetHexColor(0xffffff);  
  
	// we could draw the whole contour finder  
	//contourFinder.draw(360,540);  
  
	// or, instead we can draw each blob individually,  
	// this is how to get access to them:  
    for (int i = 0; i < contourFinder.nBlobs; i++){  
        contourFinder.blobs[i].draw(360,540);  
    }*/  
  
	// finally, a report:  
  
	//ofSetHexColor(0xffffff);  
	//char reportStr[1024];  
	//sprintf(reportStr, "bg subtraction and blob detection\npress ' ' to capture bg\nthreshold %i (press: +/-)\nnum blobs found %i, fps: %f", threshold, contourFinder.nBlobs, ofGetFrameRate());  
	//ofDrawBitmapString(reportStr, 20, 600);  
  
}  
  
//--------------------------------------------------------------  
void testApp::keyPressed(int key){  
  
	switch (key){  
		case ' ':  
			bLearnBakground = true;  
			break;  
		case '+':  
			threshold ++;  
			if (threshold > 255) threshold = 255;  
			break;  
		case '-':  
			threshold --;  
			if (threshold < 0) threshold = 0;  
			break;  
	}  
}  
  
//--------------------------------------------------------------  
void testApp::keyReleased(int key){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseMoved(int x, int y ){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseDragged(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mousePressed(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseReleased(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::windowResized(int w, int h){  
  
}  
  
//--------------------------------------------------------------  
void testApp::gotMessage(ofMessage msg){  
  
}  
  
//--------------------------------------------------------------  
void testApp::dragEvent(ofDragInfo dragInfo){   
  
}  
  

The Lucas-Kanade doesn’t complain about the sizes and all works well. I can post those . and .cpp if needed.

Any idea why it thinks my 2 inputs are different sizes? Once I transfer the colorImg to grayImg it should be single channel, right?

I don’t know much about that one (I usually use LK), but the block size and shift size mean different things than I thought they would, might be causing an error in the openCV guts. I’m wondering if the interface to that method has changed since they wrote their wrapper?

The only working example I can find online has the maxRange smaller than the blocksize, where you have it equal (http://tech.groups.yahoo.com/group/OpenCV/message/59176). Might try tweaking that?

Negative. I’m actually trying to get a down and dirty, fast global camera motion. Or at least the direction and average velocity of pixels near the left and right edges. The LKpyr method I have working is slow, but maybe I can down sample for the the values I need. I might not need the pyramid technique either, it was just one I found that works.

I’ll keep digging.

i haven’t tried but using FAST http://www.edwardrosten.com/work/fast.html- which is now included in opencv (take a look at the features2d module headers) to find features and then using LK over that features only would be way faster and probably accurate than trying to do it over the whole image

I’d agree with Arturo, you don’t want to process the whole image if possible, and camera motion shouldn’t require you to process every block of the image. I’d imagine that you might even be able to do a Hough transform on the image and then run LK on that, but FAST could be worth looking into too.

Thanks, guys. Reading up on it now.

So,

  
  
std::vector<cv::KeyPoint> keyPoints;  
cv::FAST(grayImage.getCvImage(), keyPoints,30,true);  
  

would find corner features and stick them into a vector called keyPoints, right? How do I look at those keyPoints with LKpyr? Do I need to draw them to an image then LK that?

I’m not finding much on the web about using FAST. Any help would be great.

I might be wrong, but I think you need to just pass the keyPoints from the successive call to cv::FAST to the LK.

  
void calcOpticalFlowPyrLK(prevImg.getCVImage(), curImg.getCVImage(), prevKeyPoints, keyPoints, status, err);  

some almost-helpful info is here: http://opencv.willowgarage.com/documentation/cpp/motion-analysis-and-object-tracking.html

you can also check out the lkdemo.cpp in samples/cpp if you have one of the newer versions of the OpenCV download laying aorund.

Hmmm, yes, but cv::FAST requires a vector of cv::KeyPoints while cv::calcOpticalFlowPyrLK requires it’s points to be a vector of cv::Point2f.

Guess I’ll need to convert one vector to another?

cv::KepPoint::convert! woot!

Ok, so I have the points being generated with FAST and then handed off to pyrLK.

  
cv::Size winSize(10,10);  
	std::vector<cv::KeyPoint> keyPoints;  
	std::vector<cv::KeyPoint> nextPoints;  
	cv::FAST(grayBg.getCvImage(), keyPoints,30,true);  
	cv::FAST(grayImage.getCvImage(), nextPoints,30,true);  
	// convert vector of keypoints to vector of points  
	vector<cv::Point2f> points_keyPoints, points_nextPoints;  
	cv::KeyPoint::convert(keyPoints, points_keyPoints);  
	cv::KeyPoint::convert(nextPoints, points_nextPoints);  
	cv::TermCriteria termcrit(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS,20,0.03);  
	//cv::goodFeaturesToTrack(grayImage.getCvImage(), keyPoints, MAX_COUNT, 0.01, 10, cv::Mat(), 3, 0, 0.04);  
	//cv::cornerSubPix(grayImage.getCvImage(), keyPoints, winSize, cv::Size(-1,-1), termcrit);  
	cv::vector<uchar> status;  
    cv::vector<float> err;  
	cv::calcOpticalFlowPyrLK(grayBg.getCvImage(),grayImage.getCvImage(),points_keyPoints,points_nextPoints,status, err, winSize, 3, termcrit, 0);  
	//flow.updateFlowData(grayImage.getCvImage());  
	grayBg = grayImage;  

ignore the comments, they’re from doing it a different way.

What I really need now is to reach in and extract the velocity of certain points. Ideally the average of the velocities along the left, right , and top edges.

Progress!

Hmmm. None of my point vectors seem to be picking up any points. Am I missing something?

Ok, it’s working now. cv::FAST to CV::calcOpticalFlowPyrLK

  
cv::Size winSize(5,5);  
	vector<cv::KeyPoint> keyPoints;  
	vector<cv::KeyPoint> nextPoints;  
	cv::FAST(grayBg.getCvImage(), keyPoints,30,true);  
	cv::FAST(grayImage.getCvImage(), nextPoints,30,true);  
	// convert vector of keypoints to vector of points  
	//vector<cv::Point2f> points_keyPoints, points_nextPoints;//these are in the .h now.  
	cv::KeyPoint::convert(keyPoints, points_keyPoints);  
	cv::KeyPoint::convert(nextPoints, points_nextPoints);  
	cv::TermCriteria termcrit(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS,20,0.03);  
	vector<uchar> status;  
    vector<float> err;  
  
	cv::calcOpticalFlowPyrLK(grayBg.getCvImage(),grayImage.getCvImage(),points_keyPoints,points_nextPoints, status, err, winSize, 3, termcrit, 0.5, 1);  
	  
	grayBg = grayImage;  

And to check the flow,

  
grayImage.draw(0,0);  
	ofSetColor(0xffffff);  
	int pointCount = points_nextPoints.size();  
	for( int i=0; i < points_nextPoints.size(); i++ ) {  
		ofNoFill();  
		ofSetHexColor(0xFF0000);  
		  
		ofCircle(points_nextPoints[i].x,points_nextPoints[i].y,2);  
	}  
        ofDrawBitmapString("pointCount: " + ofToString(pointCount),20,20);  
  

Now onto velocities where I need them.

i’m trying to implement to your, but i can’t get it to compile.

i keep getting undefined reference to buildpyramid.
“lkpyramid.cpp(.text._ZN2cv20calcOpticalFlowPyrLKERKNS_3MatES2_RKSt6vectorINS_6Point_IfEESaIS5_EERS7_RS3_IhSaIhEERS3_IfSaIfEENS_5Size_IiEEiNS_12TermCriteriaEdi+0x448) || undefined reference to `cv::buildPyramid(cv::Mat const&, std::vector<cv::Mat, std::allocatorcv::Mat >&, int)’|”

i’m running of7 on ubuntu 11.04, and I simply modified the opencv example to include novy san’s code.

any ideas?

thanks
bruno

bruno what version of opencv are you running?