Black texture when applying a shader using fbo

beginners
#1

Hello,

I try to apply a shader filters taken from ofxFilterLibrary on a texture using Fbo but i have a black texture instead of the texture with the shader applied.
https://github.com/mfargo/ofxFilterLibrary

I can’t solve this problem myself, i spent a few days on it but i’m stuck on this :frowning:
Here is my code, sorry it’s a bit long (i have tried to keep only the interresting parts)

The most interresting part i think is in Ornament.cpp::update.

main.cpp :

int main()
{
	ofSetupOpenGL(1280, 768, OF_WINDOW);
	ofRunApp(new ofApp());
}

ofApp.cpp :

void ofApp::setup()
{
    ornament.setup(ofGetWidth() * ZOOM_FACTOR, ofGetHeight() * ZOOM_FACTOR);

    ofLoadImage(preview, "img/mytexture.png");
    ornament.loadTexture(preview);
}

void ofApp::draw()
{
    int w = ofGetWindowWidth() * ZOOM_FACTOR;
    int h = ofGetWindowHeight() * ZOOM_FACTOR; 
    ornament.draw(-w / 2, -h / 2);
}

void ofApp::update()
{
   ornament.update();
}

Ornament.cpp :

void Ornament::setup(int w, int h, WALLPAPER_GROUP wallpaperGroup_, int tileSize_, float angle_)
{
    width = w;
    height = h;
    wallpaperGroup = wallpaperGroup_;
    tileSize = tileSize_;
    angle = angle_;
    fbo.allocate(width, height);

    ornamentShader.setupShaderFromSource(GL_VERTEX_SHADER, getVertexShader());
    ornamentShader.setupShaderFromSource(GL_FRAGMENT_SHADER, getOrnamentShader());
    ornamentShader.linkProgram();
    
    wallpaperShader.setupShaderFromSource(GL_VERTEX_SHADER, getVertexShader());
    wallpaperShader.setupShaderFromSource(GL_FRAGMENT_SHADER, getWallpaperShader());
    wallpaperShader.linkProgram();

    // INIT SHADER BASED FILTER TAKEN FROM OFXFILTERLIBRARY
    postProcessFilter = new ToonFilter(w, h);
}

void Ornament::loadTexture(ofTexture texture){
    inputTexture = texture;
    //inputTexture.setAnchorPercent(0.5, 0.5);

    inputTexture_preview = texture;
    inputTexture_preview.setAnchorPercent(0.5, 0.5);
    
    int w = width;
    int h = height;
    if (texture.getWidth() > w) {
        w = texture.getWidth();
    }
    if (texture.getHeight() > h) {
        h = texture.getHeight();
    }

    if (!tileFbo.isAllocated() || tileFbo.getWidth() != w || tileFbo.getHeight()!=h) {
        tileFbo.allocate(w, h);
        resizeFbo.allocate(w, h);
    }
    createOrnament();
}

void Ornament::update()
{
    //resize texture
    resizeFbo.begin();
    ofClear(0, 0, 0, 0);
    ofVec2f pNew, sizeNew;
    getBoundingBox(tile->getOrnamentBase(), pNew, sizeNew);
    ofVec2f sizeTex = resize(ofVec2f(inputTexture.getWidth(), inputTexture.getHeight()), sizeNew);
    ofVec2f posTex = (pNew + sizeNew * 0.5) - sizeTex * 0.5;


    ofSetMatrixMode(OF_MATRIX_TEXTURE);
    ofPushMatrix();
    ofMatrix4x4 m;
    m.glRotate(tileRotation);// 10, 1, 1, 1);
    m.glScale(tileScale);// 0.5, 1, 1);
    m.glTranslate(tilePosition);// 100, 0, 0);
    ofMultMatrix(m);


    inputTexture.draw(posTex.x, posTex.y, sizeTex.x, sizeTex.y);

    ofPopMatrix();
    ofSetMatrixMode(OF_MATRIX_MODELVIEW);

    resizeFbo.end();


    //create tile
    tileFbo.begin();
    ofClear(0, 0, 0, 0);
    ornamentShader.begin();
    ornamentShader.setUniform1i("cell_structure", tile->getCellStructure());
    ornamentShader.setUniform1i("wallpaper_group", wallpaperGroup);
    ornamentShader.setUniform1f("width", inputTexture.getWidth());
    ornamentShader.setUniform1f("height", inputTexture.getHeight());
    resizeFbo.draw(0, 0);
    ornamentShader.end();
    tileFbo.end();


    //create wallpaper
    fbo.begin();
    ofClear(255, 255, 255, 0);
    ofSetColor(255, 255, 255, 255);
    wallpaperShader.begin();
    wallpaperShader.setUniform1i("cell_structure", tile->getCellStructure());
    wallpaperShader.setUniform1f("width", width);
    wallpaperShader.setUniform1f("height", height);
    wallpaperShader.setUniform1f("tile_size", tileSize);
    wallpaperShader.setUniform1f("angle", angle);
    wallpaperShader.setUniform1f("tex_width", inputTexture.getWidth());
    wallpaperShader.setUniform1f("tex_height", inputTexture.getHeight());
    tileFbo.draw(0, 0);
    wallpaperShader.end();
    fbo.end();

    // try to apply ofxFilterLibrary ToonShader
    // PROBLEM OCCURS HERE
    postProcessFilter->begin(); // BLACK TEXTURE BECAUSE OF THAT LINE !
    fbo.draw(0, 0);
    postProcessFilter->end(); // BLACK TEXTURE BECAUSE OF THAT LINE !

    outputTexture = fbo.getTexture();

    tile->updateTile();
}

void Ornament::draw(int x, int y)
{
    outputTexture.draw(x, y);
}

string Ornament::getVertexShader(){
    string out = string("#version 120\n") +
    STRINGIFY(
              void main() {
                  gl_TexCoord[0] = gl_MultiTexCoord0;
                  gl_Position = ftransform();
              }
              );
    return out;
}

string Ornament::getOrnamentShader(){
    string out = string("#version 120\n#extension GL_ARB_texture_rectangle : enable\n#define PI 3.141592653589793\n#define P4 9\n#define P4M 10\n#define P4G 11\n#define P3 12\n#define P3M1 13\n#define P31M 14\n#define P6 15\n#define P6mm 16\n") +
    STRINGIFY(
              uniform sampler2DRect 	u_tex_unit0;
              uniform float 			width,height;
              uniform int 			wallpaper_group;
              /*
               0 = P1,
               1 = P2,
               2 = PM,
               3 = PG,
               4 = CM,
               5 = PMM,
               6 = PMG,
               7 = PGG,
               8 = CMM
               */
              uniform int 			cell_structure;
              /*
               0 = RECTANGLE,
               1 = SQUARE,
               2 = RHOMBIC,
               3 = HEXAGONAL,
               4 = OBLIQUE
               */
              
              vec2 p1,p2,p3,p4;
              vec2 base1,base2,base3,base4,mid;
              vec2 p1p2, p2p3, p3p4, p1p4;
              
              /*****************
               * Transformation *
               ******************/
              vec2 translate(vec2 point ,vec2 trans){
                  return point + trans;
              }
              
              vec2 rotate(vec2 point, float angle){
                  mat2 rot = mat2 (
                                   cos(angle), sin(angle),
                                   -sin(angle),cos(angle));
                  return rot * point;
              }
              
              vec2 rotateAt(vec2 point, vec2 anchor, float angle){
                  return vec2(
                              cos(angle)*point.x - sin(angle)*point.y + anchor.x - cos(angle)*anchor.x + sin(angle)*anchor.y,
                              sin(angle)*point.x + cos(angle)*point.y + anchor.y - sin(angle)*anchor.x - cos(angle)*anchor.y);
              }
              
              vec2 reflectAt(vec2 point, vec2 anchor, vec2 dir){
                  vec2 I = point-anchor;
                  vec2 N = normalize(dir);
                  vec2 r = reflect(I,N);
                  return anchor - r;
              }
              
              /************************
               * Position calculation *
               ************************/
              void getBorders(){
                  // CUTTED BECAUSE TOO LONG
              }
              bool PointInTriangle(vec2 p, vec2 p0, vec2 p1, vec2 p2)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              bool isInTile(vec2 p)
              {
                  //pt resolves edge jitter
                  vec2 pt = vec2(p.x-0.1,p.y-0.1);
                  if(PointInTriangle(p,p1,p2,p4) || PointInTriangle(p,p2,p3,p4)
                     ||PointInTriangle(pt,p1,p2,p4) || PointInTriangle(pt,p2,p3,p4)){
                      return true;
                  }
                  return false;
              }
              
              //type 0=P4, 1=P4M, 2=P4G
              int getTileSquare(vec2 pos,int type){
                  // CUTTED BECAUSE TOO LONG
              }
              
              int getTileTriangle(vec2 pos,int type){
                  //ornament base
                  base1 = p1;
                  vec2 p1p2 = p1 + (p2-p1)/2;
                  base2 = p1p2 + (p3-p1p2)/3;
                  base3 = p3;
                  vec2 p1p4 = p1 + (p4-p1)/2;
                  base4 = p1p4 + (p3-p1p4)/3;
                  
                  int t1,t2,t3;
                  
                  if(PointInTriangle(pos,p1,p3,p4)){
                      t1 = 0;
                      if(PointInTriangle(pos,p1,p3,base4)){
                          t2 = 0;
                          if (PointInTriangle(pos,p1,p1 + (p3-p1)/2,base4)) t3 = 0;
                          else t3 = 1;
                      }else if(PointInTriangle(pos,p4,p3,base4)){
                          t2 = 1;
                          if (PointInTriangle(pos,p3,p4 + (p3-p4)/2,base4)) t3 = 0;
                          else t3 = 1;
                      }
                      else{
                          t2 = 2;
                          if (PointInTriangle(pos,p4,p4 + (p1-p4)/2,base4)) t3 = 0;
                          else t3 = 1;
                      }
                  }else{
                      t1 = 1;
                      if(PointInTriangle(pos,p1,p3,base2)){
                          t2 = 0;
                          if (PointInTriangle(pos,p1,p1 + (p3-p1)/2,base2)) t3 = 0;
                          else t3 = 1;
                      }else if(PointInTriangle(pos,p1,p2,base2)){
                          t2 = 1;
                          if (PointInTriangle(pos,p2,p1 + (p2-p1)/2,base2)) t3 = 0;
                          else t3 = 1;
                      }
                      else{
                          t2 = 2;
                          if (PointInTriangle(pos,p3,p3 + (p2-p3)/2,base2)) t3 = 0;
                          else t3 = 1;
                      }
                  }
                  int t = t1*6 + t2*2 + t3;
                  
                  int lut[12];
                  if(type == 0){
                      // 000 001 010 011 020 021 100 101 110 111 120 121
                      lut = int[12]( 0,  0,  3,  3,  4,  4,  0,  0,  1,  1,  2,  2);
                  }else if(type == 1){
                      // 000 001 010 011 020 021 100 101 110 111 120 121
                      lut = int[12]( 0, 1, 31, 30, 40, 41,  0,  1, 11, 10,  20, 21);
                  }else if(type == 2 || type == 3){
                      // 000 001 010 011 020 021 100 101 110 111 120 121
                      lut = int[12]( 0,  0,  4,  4,  5,  5,  1,  1,  2,  2,  3,  3);
                  }else if(type == 4){
                      // 000 001 010 011 020 021 100 101 110 111 120 121
                      lut = int[12]( 0,  1,  40,  41,  50,  51,  10,  11,  20,  21,  30,  31);
                  }
                  
                  return lut[t];
              }
              /*************************************
               * WALLPAPER RENDERING  P4 Groups*
               ************************************/
              void do_p4(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void do_p4m(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void do_p4g(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              /*************************************
               * WALLPAPER RENDERING  P3, P6 Groups*
               ************************************/
              void do_p3(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void do_p3m1(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void do_p31m(vec2 pos)
              {
                 // CUTTED BECAUSE TOO LONG
              }
              
              void do_p6(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void do_p6mm(vec2 pos)
              {
                  // CUTTED BECAUSE TOO LONG
              }
              
              void main( void )
              {
                  getBorders();
                  
                  vec4 color = texture2DRect( u_tex_unit0, gl_TexCoord[0].st );
                  vec2 pos = gl_TexCoord[0].st;
                  
                  //gl_FragColor = texture2DRect( u_tex_unit0, rotateAt (pos,vec2(width/2,height/2),cos(u_time) ));
                  if(isInTile(pos)){
                      if (wallpaper_group == P4) do_p4(pos);
                      else if (wallpaper_group == P4M) do_p4m(pos);
                      else if (wallpaper_group == P4G) do_p4g(pos);
                      else if (wallpaper_group == P3) do_p3(pos);
                      else if (wallpaper_group == P3M1) do_p3m1(pos);
                      else if (wallpaper_group == P31M) do_p31m(pos);
                      else if (wallpaper_group == P6) do_p6(pos);
                      else if (wallpaper_group == P6mm) do_p6mm(pos);
                      
                  }else{
                      gl_FragColor = vec4(0.0,0.0,0.0,0.0);
                  }
                  
                  
              }              );
    return out;
}

string Ornament::getWallpaperShader(){
    string out = string("#version 120\n#extension GL_ARB_texture_rectangle : enable\n#define PI 3.141592653589793\n#define RECTANGLE 0\n#define SQUARE 1\n#define RHOMBIC 2\n#define HEXAGONAL 3\n#define OBLIQUE 4\n") +
    STRINGIFY(uniform sampler2DRect 	u_tex_unit0;
              uniform float           tex_width,tex_height;
              uniform float 			width,height;
              uniform int 			cell_structure;
              /*
               0 = RECTANGLE,
               1 = SQUARE,
               2 = RHOMBIC,
               3 = HEXAGONAL,
               4 = OBLIQUE
               */
              uniform float 			tile_size;
              uniform float			angle;
              
              
              
              vec2 p1,p2,p3,p4;
              vec2 base1,base2,base3,base4;
              vec2 nextCellSouth, nextCellEast, nextCellNorth, nextCellWest;
              
              /*****************
               * Transformation *
               ******************/
              float map(float val,float inMin,float inMax,float outMin,float outMax){
                  return outMin + ((outMax - outMin) / (inMax - inMin)) * (val - inMin);
              }
              
              float fmod(float val, float modulo){
                  int i = int(val/modulo);
                  return val - (i*modulo);
              }
              
              vec2 translate(vec2 point ,vec2 trans){
                  return point + trans;
              }
              
              vec2 rotate(vec2 point, float angle){
                  mat2 rot = mat2 (
                                   cos(angle), sin(angle),
                                   -sin(angle),cos(angle));
                  return rot * point;
              }
              
              vec2 rotateAt(vec2 point, vec2 anchor, float angle){
                  return vec2(
                              cos(angle)*point.x - sin(angle)*point.y + anchor.x - cos(angle)*anchor.x + sin(angle)*anchor.y,
                              sin(angle)*point.x + cos(angle)*point.y + anchor.y - sin(angle)*anchor.x - cos(angle)*anchor.y);
              }
              
              vec2 reflectAt(vec2 point, vec2 anchor, vec2 dir){
                  vec2 I = point-anchor;
                  vec2 N = normalize(dir);
                  vec2 r = reflect(I,N);
                  return anchor - r;
              }
              
              bool PointInTriangle(vec2 p, vec2 p0, vec2 p1, vec2 p2)
              {
                  float s = p0.y * p2.x - p0.x * p2.y + (p2.y - p0.y) * p.x + (p0.x - p2.x) * p.y;
                  float t = p0.x * p1.y - p0.y * p1.x + (p0.y - p1.y) * p.x + (p1.x - p0.x) * p.y;
                  
                  if ((s < 0) != (t < 0))
                      return false;
                  
                  float A = -p1.y * p2.x + p0.y * (p2.x - p1.x) + p0.x * (p1.y - p2.y) + p1.x * p2.y;
                  if (A < 0.0)
                  {
                      s = -s;
                      t = -t;
                      A = -A;
                  }
                  return s > -0.01 && t > -0.01 && (s + t) < A;
              }
              
              /************************
               * Position calculation *
               ************************/
              
              void updateCornerPoints(){
                  // CUTTED BECAUSE TOO LONG
              }
              
              vec2 do_square(vec2 pos){
                  // CUTTED BECAUSE TOO LONG
              }
              
              vec2 do_hexagonal(vec2 pos){
                  // CUTTED BECAUSE TOO LONG
              }
              
              void main( void )
              {
                  updateCornerPoints();
                  
                  vec4 color = texture2DRect( u_tex_unit0, gl_TexCoord[0].st );
                  vec2 pos = gl_TexCoord[0].st;
                  
                  pos = rotateAt(pos,vec2(width/2,height/2),angle);
                  if(cell_structure == SQUARE){
                      pos = do_square(pos);
                  }
                  if(cell_structure == HEXAGONAL){
                      pos = do_hexagonal(pos);
                  }
                  
                  vec4 c = texture2DRect(u_tex_unit0, pos);
                  gl_FragColor = c;
              }
         );
    return out;
}

Thanks a lot for your help !
I really need to solve that issue :slight_smile:

Eviral

#2

Hey @Eviral , can you see something reasonable if you were to draw fbo at this point (the end of the wallpaperShader pass) in Ornament::draw()? This would make sure that ToonFilter has something meaningful to work with in fbo.

I looked at the ofxFilterLibrary example and the addon classes for a bit. It seems like postProcessFilter is created and used in a way that is similar to the example. However, the example has some lines in ofApp:draw() (not ofApp::update()) that are similar to these lines in Ornament::update():

    postProcessFilter->begin(); // BLACK TEXTURE BECAUSE OF THAT LINE !
    fbo.draw(0, 0);
    postProcessFilter->end(); // BLACK TEXTURE BECAUSE OF THAT LINE !

Have you tried rendering into an ofFbo here? Or alternatively, moving these calls into Ornament::draw()?

Sometimes it helps to write a very very simple example to test and make sure things are working OK. If you haven’t already, you could do this with a ToonFilter instance in ofApp, with only the essential elements that would be used in a project, and maybe an ofImage for it to chew on.

Finally, you can try writing a toonShader shader, which would be setup and used the same way as ornamentShader and wallpaperShader, but that uses the shader code (and variables) from ToonFilter.cpp.

#3

Hello,

Thanks a lot for your answer…

Hey @Eviral , can you see something reasonable if you were to draw fbo at this point (the end of the wallpaperShader pass) in Ornament::draw()? This would make sure that ToonFilter has something meaningful to work with in fbo.

I have already tried, same result : black texture

I looked at the ofxFilterLibrary example and the addon classes for a bit. It seems like postProcessFilter is created and used in a way that is similar to the example. However, the example has some lines in ofApp:draw() (not ofApp::update()) that are similar to these lines in Ornament::update():
Have you tried rendering into an ofFbo here? Or alternatively, moving these calls into Ornament::draw()?

Also tried, same result

Sometimes it helps to write a very very simple example to test and make sure things are working OK. If you haven’t already, you could do this with a ToonFilter instance in ofApp, with only the essential elements that would be used in a project, and maybe an ofImage for it to chew on.

I have just uploaded a simple example reproducing the problem here :
https://github.com/flarive/BlackTextureIssue
Could you have a look please ?

Finally, you can try writing a toonShader shader, which would be setup and used the same way as ornamentShader and wallpaperShader, but that uses the shader code (and variables) from ToonFilter.cpp.

No, i would like to let user choose any filter he wants from ofxFilterLibrary.
ToonFilter is just a test filter i have choosen to describe this issue.

Thanks a lot
I really appreciate your help :slight_smile:

#4

Ah OK this makes sense. I have a feeling you’ll have to use an ofFbo for this, or use it during ofApp::draw().

So just to confirm, there is nothing in fbo after wallpaperShader.end(), correct? If this is the case, then how about tileFbo, resizeFbo, and inputTexture? Using unique ofFbos like this can help troubleshoot. It can be tedious, but working thru the chain and evaluating (drawing) the output (and sometimes input) of each shader can help localize the problem.

Sometimes I’ll set a temporary output value in a shader just to make sure its working, something like:

vec2 pos = gl_TexCoord[0].st; // maybe normalize this depending
gl_FragColor = vec4(pos.x, 0.0, pox.y, 1.0);

And sometimes I’ll pass a “known texture” to a shader (like an ofImage) just to make sure its sampling and calculating correctly.

Black screens in shaders can arise from not setting a uniform, or having a shader variable (or #define) set to a “wrong value”, such that the shader calculates values <= 0.0. They can also arise from uncompiled shaders in a chain, but I’ve always seen error output in the IDE when they don’t compile.

I’m not great with shaders that use the internal variables like gl_TexCoord and gl_FragColor. Potentially, a black screen could arise from sampling a non-normalized texture with normalized coordinates if the first pixel is black. So another quick troubleshooting tactic is to quickly check (or set) the color of this pixel before passing the texture to the shader.

Sometimes it helps to start with a small, simple program, and then slowly test and expand the complexity of it by adding 1 shader to the chain at a time and then confirming the output.

#5

Thanks !
May be it can helps, but if I use some others ofxFilterLibrary filter (instead of ToonFilter but don’t remember which) i don’t have a black texture but the original texture without any shader applied.

Strange behavior, it seems to depend on the filter…

#6

Another strange thing to note…
If i use the ShaderBlurX.vert/ShaderBlurX.frag shader found in OF exemples/shader/09_gaussianBlurFilter instead of a ofxFilterLibrary filter it works well (no black texture, blur is applied like a charm)

I have also tested ofxFilterLibrary filters on a simple ofImage and it works perfectly…

Very strange issue…

#7

I’m cautiously wondering if this all might be an issue with normalized texture coordinates. The ofxFilterLibrary example calls ofDisableArbTex() in ofApp::setup(), which (I’m thinking) results in normalized values for gl_TexCoord[0]. But again I’m not great with these internal variables and how they work.