HS flow shaders

Hi All,

I grabbed the HS-flow shaders out of the v002 Optical Flow quartz plugin http://v002.info/plugins/v002-optical-flow/ and wanted to run them in oF but I’m running into some difficulty getting them going.

It seems as if there are two sets of shaders at play here. As far as I can tell, the first shader calculates the motion between frames and recolors the pixel based on that analysis. The second shader looks at the output of the first one, and uses that difference to distort the pixels. I tried setting this up with each shader in it’s own FBO.

The first shader runs fine (I think). I end up with a mostly RGB and black image, that almost looks like an edge detection output. But when I pass that image into the second FBO and shader, I get a black screen.

I’m also a little uncertain about setting the texture level int, I assume that number the same as the position in the gl_TextureMatrix[]?

Here’s what I’ve got:

The vertex shaders are both the same, pretty standard

  
  
varying vec2 texcoord0;  
varying vec2 texcoord1;  
  
void main()  
{  
    // perform standard transform on vertex  
    gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;  
  
    // transform texcoords  
    texcoord0 = vec2(gl_TextureMatrix[0] * gl_MultiTexCoord0);  
  
	// there is always something weird with gl_MultiTexCoord > 0... :(  
	texcoord1 = vec2(gl_TextureMatrix[1] * gl_MultiTexCoord1);  
}  
  

The first fragment shader “flow.frag”

  
  
//Andrew Benson - andrewb@cycling74.com  
//2009  
  
// texcoords  
varying vec2 texcoord0;  
varying vec2 texcoord1;  
  
// samplers  
uniform sampler2DRect tex0;  
uniform sampler2DRect tex1;  
  
//variables  
uniform vec2 scale;  
uniform vec2 offset;  
uniform float lambda;  
const vec4 lumcoeff = vec4(0.299,0.587,0.114,0.);  
  
// entry point  
void main()  
{     
	vec4 a = texture2DRect(tex0, texcoord0);  
	vec4 b = texture2DRect(tex1, texcoord1);  
	vec2 x1 = vec2(offset.x,0.);  
	vec2 y1 = vec2(0.,offset.y);  
  
	//get the difference  
	vec4 curdif = b-a;  
	  
	//calculate the gradient  
      
    vec4 gradx = texture2DRect(tex1, texcoord1+x1)-texture2DRect(tex1, texcoord1-x1);  
	gradx += texture2DRect(tex0, texcoord0+x1)-texture2DRect(tex0, texcoord0-x1);  
	  
	vec4 grady = texture2DRect(tex1, texcoord1+y1)-texture2DRect(tex1, texcoord1-y1);  
	grady += texture2DRect(tex0, texcoord0+y1)-texture2DRect(tex0, texcoord0-y1);  
	  
	vec4 gradmag = sqrt((gradx*gradx)+(grady*grady)+vec4(lambda));  
  
	vec4 vx = curdif*(gradx/gradmag);  
	float vxd = vx.r;//assumes greyscale  
	//format output for flowrepos, out(-x,+x,-y,+y)  
	vec2 xout = vec2(max(vxd,0.),abs(min(vxd,0.)))*scale.x;  
  
	vec4 vy = curdif*(grady/gradmag);  
	float vyd = vy.r;//assumes greyscale  
	//format output for flowrepos, out(-x,+x,-y,+y)  
	vec2 yout = vec2(max(vyd,0.),abs(min(vyd,0.)))*scale.y;  
  
	//gl_FragColor = vec4(yout.xy,yout.xy);  
	  
	gl_FragColor = vec4(xout.xy,yout.xy);  
      
}  
  

The second fragment shader “repos.frag”

  
  
//setup for 2 texture  
varying vec2 texcoord0;  
varying vec2 texcoord1;  
//varying vec2 texdim0;  
//varying vec2 texdim1;  
uniform vec2 amt;  
//uniform vec4 scale;  
//uniform vec4 bias;  
uniform vec2 boundmode;  
uniform sampler2DRect tex0;  
uniform sampler2DRect tex1;  
  
  
void main()  
{  
    vec4 look = texture2DRect(tex1,texcoord1);//sample repos texture  
    vec2 offs = vec2(look.y-look.x,look.w-look.z)*amt;  
    vec2 coord = offs+texcoord0;	//relative coordinates  
    //coord = mod(coord,texdim0);  
    vec4 repos = texture2DRect(tex0, coord);  
      
    // output texture  
    gl_FragColor = repos;//*scale+bias;  
}  
  

.h

  
  
#pragma once  
  
#include "ofMain.h"  
  
class testApp : public ofBaseApp{  
	public:  
		void setup();  
		void update();  
		void draw();  
		  
		void keyPressed(int key);  
		void keyReleased(int key);  
		void mouseMoved(int x, int y);  
		void mouseDragged(int x, int y, int button);  
		void mousePressed(int x, int y, int button);  
		void mouseReleased(int x, int y, int button);  
		void windowResized(int w, int h);  
		void dragEvent(ofDragInfo dragInfo);  
		void gotMessage(ofMessage msg);  
          
        ofShader        flow;  
        ofShader        repos;  
      
        ofImage         prev;  
        ofVideoPlayer   movie;  
      
        int w;   
        int h;  
      
        ofFbo fboFlow;  
        ofFbo fboRepos;  
};  
  

.cpp

  
  
#include "testApp.h"  
  
//--------------------------------------------------------------  
void testApp::setup(){  
    ofSetVerticalSync(true);  
      
    flow.load("flow.vert","flow.frag");  
    repos.load("repos.vert","repos.frag");  
      
    w = 720;  
    h = 480;  
      
    ofSetWindowShape(w,h);  
      
    movie.setUseTexture(true);  
    movie.loadMovie("movie.mov");  
    movie.play();  
      
    prev.allocate(w, h, OF_IMAGE_COLOR);  
      
    fboFlow.allocate(w, h);  
    fboRepos.allocate(w,h);  
      
    fboFlow.begin();  
    ofClear(0);  
    fboFlow.end();  
      
    fboRepos.begin();  
    ofClear(0);  
    fboRepos.end();  
}  
  
//--------------------------------------------------------------  
void testApp::update(){  
  
}  
  
//--------------------------------------------------------------  
void testApp::draw(){  
      
    unsigned char* videoPix = movie.getPixels();  
      
    fboFlow.begin();  
    flow.begin();  
      
    movie.update();  
      
    flow.setUniform2f("scale", 255, 255);  
    flow.setUniform2f("offset", 1.0, 1.0);  
      
    flow.setUniform1f("lambda", ofGetElapsedTimef());  
    flow.setUniformTexture("tex0", movie.getTextureReference(), 0);  
    flow.setUniformTexture("tex1", prev.getTextureReference(), 1);  
      
    movie.draw(0,0);  
      
    flow.end();  
    fboFlow.end();  
      
    //grab previous pixels  
    prev.setFromPixels(videoPix, w, h, OF_IMAGE_COLOR);  
      
      
    fboRepos.begin();  
    repos.begin();  
      
      
    repos.setUniformTexture("tex1", fboFlow.getTextureReference(), 3);  
    repos.setUniformTexture("tex0", movie.getTextureReference(), 2);  
    repos.setUniform2f("amt", 1.0, 1.0);  
      
    repos.end();  
    fboRepos.end();  
      
    fboRepos.draw(0,0);  
    //fboFlow.draw(0, 0);  
}  
  
//--------------------------------------------------------------  
void testApp::keyPressed(int key){  
  
}  
  
//--------------------------------------------------------------  
void testApp::keyReleased(int key){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseMoved(int x, int y){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseDragged(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mousePressed(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::mouseReleased(int x, int y, int button){  
  
}  
  
//--------------------------------------------------------------  
void testApp::windowResized(int w, int h){  
  
}  
  
//--------------------------------------------------------------  
void testApp::gotMessage(ofMessage msg){  
  
}  
  
//--------------------------------------------------------------  
void testApp::dragEvent(ofDragInfo dragInfo){   
  
}  
  

Uncommenting the fboFlow.draw will give you an idea of what the first shader is doing.

I might just be totally misunderstanding what the purpose of these shaders is, but essentially what I’m after is a real-time optical flow distortion somewhat similar to the one in the ofxcv examples.

Any help is greatly appreciated! Thanks!

Ok, so I’ve been slowly chipping away at this. I’ve made a little progress, but now am stuck again, though I feel the end is close.

The shaders are unchanged from the previous post, but I started clean again with my .cpp and .h. I am referencing this post for some of the uniform floats: https://groups.google.com/forum/#!msg/jitterinmax4live-/zWKLTzmUvzI/dQuh63AsT2AJ I realized that the flow shader is actually looking for a grayscale image, and that the two shaders work by using a feedback loop on the gpu. I’m getting a little confused with which inputs go to which outputs. If you run this it will produce almost the desired effect, but instead of showing the repos shader on the webcam pixels, it shows it on the flow shader output. I can’t quite figure out how to get the feedback to affect the webcam output. I’ve tried swapping around all of the uniformTextures but I feel like I’m just going in circles. Hoping someone can take a quick look and see if perhaps I’m missing something or making a mistake.

I’m also not sure if this is the right way to go about implementing this, or if I should be using two fbo’s instead. Still feeling a little confused about where exactly the textures are sitting (how can two textures both be occupying the 0 position?), or how get information about where they live. If I reverse the order that the shaders take place, I get a mix of the webcam + the flow shader.

.h

  
  
class testApp : public ofBaseApp{  
	public:  
		void setup();  
		void update();  
		void draw();  
		  
		void keyPressed(int key);  
		void keyReleased(int key);  
		void mouseMoved(int x, int y);  
		void mouseDragged(int x, int y, int button);  
		void mousePressed(int x, int y, int button);  
		void mouseReleased(int x, int y, int button);  
		void windowResized(int w, int h);  
		void dragEvent(ofDragInfo dragInfo);  
		void gotMessage(ofMessage msg);  
      
        ofVideoGrabber cam;  
        ofTexture grayTex, lastTex;  
        unsigned char * gray;  
        int w, h;  
        ofShader shader, hsflow;  
        ofFbo fbo, fbo2;  
};    
  

.cpp

  
  
#include "testApp.h"  
  
//--------------------------------------------------------------  
void testApp::setup(){  
    w = 600;  
    h = 400;  
          
    cam.initGrabber(w, h, true);  
    cam.setUseTexture(true);  
      
    shader.load("repos");  
    hsflow.load("flow");  
      
    gray = new unsigned char[w*h];  
    grayTex.allocate(w,h,GL_LUMINANCE);  
    lastTex.allocate(w,h,GL_LUMINANCE);  
      
    ofEnableAlphaBlending();  
    ofSetBackgroundAuto(false);  
      
    fbo.allocate(w,h,GL_RGB);  
    fbo.setUseTexture(true);  
      
    //fbo2.allocate(w,h,GL_RGB);  
    //fbo2.setUseTexture(true);  
      
    fbo.begin();  
    ofClear(0,0,0);  
    fbo.end();  
      
    //fbo2.begin();  
    //ofClear(0,0,0);  
    //fbo2.end();  
      
}  
  
//--------------------------------------------------------------  
void testApp::update(){  
    cam.update();  
}  
  
//--------------------------------------------------------------  
void testApp::draw(){  
  
    //loop through pixels and convert to a grayscale texture  
    unsigned char * pixels = cam.getPixels();  
    int counter = 0;  
    for(int y = 0; y<h; y++){  
        for(int x = 0; x<w; x++){  
            int loc = (y*w+x)*3;  
            int r = loc;  
            int g = loc+1;  
            int b = loc+2;  
            int grayPix = (11 * pixels[r]+16 * pixels[g] + 5 * pixels[b])/32;  
              
            gray[counter] = grayPix;  
            counter++;  
        }  
    }  
  
    grayTex.loadData(gray,w,h,GL_LUMINANCE);  
      
      
    fbo.begin();  
    hsflow.begin(); //flow shader  
      
    hsflow.setUniform2f("scale", 3.8, 3.8);  
    hsflow.setUniform2f("offset", 2.0,2.0);  
    hsflow.setUniform1f("lambda",0.0);  
    hsflow.setUniformTexture("tex0", grayTex, 0);  
    hsflow.setUniformTexture("tex1", lastTex, 1);  
      
    grayTex.draw(0,0);  
      
    hsflow.end();  
    fbo.end();  
      
    lastTex = grayTex;  
      
      
    fbo.begin();  
    shader.begin(); //repos shader  
      
    shader.setUniform2f("amt", 3.0,3.0);  
    shader.setUniformTexture("tex0", cam, 0);  
    shader.setUniformTexture("tex1", fbo, 1);  
      
    fbo.draw(0,0); //feed it back  
      
    shader.end();  
    fbo.end();  
  
    fbo.draw(0,0);  
      
}  
  
  

hmm searching | working on the same…

did you had any progress?

not concerning this precise problem but maybe i have general hints.

turning an image into grayscale is computer vision standard which u do on CPU…Optical flow is advanced which u already do on GPU…Why dont u do both on GPU…especially since looping through all elements…can cause more time then doin optical flow on cpu…

U can also put all drawing of FBo and updating of images from cam - into the update function…to increase speed.

Anyway here is my try and i do see sth. but i dont know if thats right

#include “testApp.h”

//--------------------------------------------------------------
void testApp::setup(){
w = 640;
h = 480;

cam.initGrabber(w, h, true);
cam.setUseTexture(true);

shader.load(“repos”);
hsflow.load(“flow”);

gray = new unsigned char[w*h];
grayTex.allocate(w,h,GL_LUMINANCE);
lastTex.allocate(w,h,GL_LUMINANCE);

ofEnableAlphaBlending();
ofSetBackgroundAuto(false);

fbo.allocate(w,h,GL_RGB);
fbo.setUseTexture(true);

fbo2.allocate(w,h,GL_RGB);
fbo2.setUseTexture(true);

fbo.begin();
ofClear(0,0,0);
fbo.end();

fbo2.begin();
ofClear(0,0,0);
fbo2.end();

ofSetBackgroundAuto(false);
}

//--------------------------------------------------------------
void testApp::update(){
cam.update();

//loop through pixels and convert to a grayscale texture
unsigned char * pixels = cam.getPixels();
int counter = 0;
for(int y = 0; y<h; y++){
for(int x = 0; x<w; x++){
int loc = (y*w+x)*3;
int r = loc;
int g = loc+1;
int b = loc+2;
int grayPix = (11 * pixels[r]+16 * pixels[g] + 5 * pixels[b])/32;

gray[counter] = grayPix;
counter++;
}
}

grayTex.loadData(gray,w,h,GL_LUMINANCE);

fbo.begin();
hsflow.begin(); //flow shader

hsflow.setUniform2f(“scale”, 3.8, 3.8);
hsflow.setUniform2f(“offset”, 2.0,2.0);
hsflow.setUniform1f(“lambda”,0.0);
hsflow.setUniformTexture(“tex0”, grayTex, 0);
hsflow.setUniformTexture(“tex1”, lastTex, 1);

grayTex.draw(0,0);

hsflow.end();
fbo.end();

lastTex.loadData(gray,w,h,GL_LUMINANCE);

fbo2.begin();
shader.begin(); //repos shader

shader.setUniform2f(“amt”, 3.0,3.0);
shader.setUniformTexture(“tex0”, cam, 0);
shader.setUniformTexture(“tex1”, fbo, 1);

fbo.draw(0,0); //feed it back

shader.end();
fbo2.end();
}

//--------------------------------------------------------------
void testApp::draw(){

grayTex.draw(0,0);
fbo.draw(640,0);
fbo2.draw(640,480);

}

Just a quick update. I did eventually figure this out!

I left the files on a dropbox folder here if anyone is interested in checking it out.

Enjoy, let me know if you can’t get it up and running.