directshow sample data layout

hi.

maybe the wrong forum for this question, but maybe zach or theo have some hints.

i wrote my own framegrabber now (based on DSVL) but i am stuck loading the grabbed sample into a opencv image or oftexture.

do you know what data layout IMediaSample::GetPointer() returns?
i can’t find anything on the web.

muchas gracias,
didi

question @ theo:

can you tell us more about the data layout for example for videoInput::getPixels()?
you wrote in the header that it "fills pixels as a BGR (for openCV) or RGB (for opengl) unsigned char array ".

but how does it fill?
R1 G1 B1, R2 G2 B1 … //line 1 - n
horizontal line by horizontal line?
or [RED CHANNEL] [GREEN CHANNEL] [BLUE CHANNEL]?

hey!
the format should be bgrbgrbgr etc

didito: Did you get your DSLV classes working? If so, can you post the source and how to use it with OF?

hey cerubcat.

yes, got it to work. where exactly are you stuck?
i have to check the code …

didito

Well I haven’t tried anything.

I just wanted to see what you did so I can compare it to the videoInput driver theo made and OF uses. Are you using DSLV instead? With our release of tbeta, we’re having some issues with cameras and not being able to access the property dialog sheet that DSLV has, so just wondering if yours would fix some of this.

yes, i am using DSVL and it works very well.
i had to apply some changes to it to compile on current visual studio, make it more safe and stable, easier to debug, etc.
and i implemented my own framegrabber class which is just my very own rewrite of the reactivision source.

i am sorry i can’t give you the whole code, but this is what the major parts look like.
i am still having a problem with my logitech webcam and DSVL. i get a nasty BSOD, i guess the directshow filter driver is somehow unstable.

header:

  
  
#ifndef __FRAMEGRABBER_HPP  
#define __FRAMEGRABBER_HPP  
  
  
#include <string>  
//put COM header(s) here  
#include "util/NoCopy.hpp"  
#include "DSVL.h"  
  
  
/*  
 * FrameGrabber class  
 * aka SampleGrabber, Sampler, ...  
 *  
 * can open camera stream (directshow)  
 * can grab current frame  
 * can be configured by xml strings -> i.e.: std::string config("<?xml version=\"1.0\" encoding=\"UTF-8\"?><dsvl_input><camera friendly_name=\"AVT 1394 camera\" show_format_dialog=\"true\"><pixel_format><RGB32/></pixel_format></camera></dsvl_input>");  
 */  
class FrameGrabber : public ext::NoCopy  
{  
public:  
    FrameGrabber(const std::string& configXmlString);  
   ~FrameGrabber();  
  
    bool  Start();  
    void  Stop(bool forceToStop = false);  
  
    bool  Initialize(const std::string& config);  
    inline bool IsInitialized() const { return isInitialized; }  
  
    void  ShowSettings() const;  
  
    void  GetCurrentFrameData(unsigned char* dst);  
    uint  GetNumberOf8BitChannels() const;  
    uint  GetWidth()                const;  
    uint  GetHeight()               const;  
  
    int   HasDroppedFrames() const;  
  
  
private:  
    void  getMediaFormatValues();  
  
  
    bool                isInitialized;  
  
    DSVL_VideoSource*   videoSource;  
    MemoryBufferHandle  mbHandle;  
    LONGLONG            timeStamp;  
  
    uint                width;  
    uint                height;  
    PIXELFORMAT         format;  
    uint                numBytes;  
    float               fps;  
  
    unsigned char*      buffer;    //opaque handle to pixels, no need to clean up!  
  
    std::string         config;    //for saving and reloading  
};  
  
  
#endif  
  

implementation:

  
  
#include <cassert>  
#include <cstdio>  
#include "FrameGrabber.hpp"  
  
  
FrameGrabber::FrameGrabber(const std::string& configXmlString)  
    : isInitialized(false)  
    , videoSource  (0)  
    , timeStamp    (0)  
    , width        (0)  
    , height       (0)  
    , format       (PIXELFORMAT_UNKNOWN)  
    , numBytes     (0)  
    , fps          (0.f)  
    , buffer       (0)  
    , config       (configXmlString)  
{  
    //put your COM initialization routine here!  
  
    videoSource = new DSVL_VideoSource();  
    assert(videoSource);  
    if (videoSource)  
    {  
        if (Initialize(configXmlString))  
        {  
            isInitialized = true;  
            printf("[INFO] DSVL based framegrabber successfully initialized\n");  
        }  
        else  
        {  
            printf("[ERROR] could NOT initialize DSVL based framegrabber with configstring:\n\t%s\n\n", configXmlString.c_str());  
        }  
    }  
    else  
    {  
        printf("[ERROR] could NOT allocate DSVL video source\n");  
    }  
}  
  
  
FrameGrabber::~FrameGrabber()  
{  
    Stop();  
    if (videoSource)  
    {  
        videoSource->DisableMemoryBuffer();  
        delete videoSource;  
        videoSource = 0;  
    }  
  
    //deinitialize COM here, but could be a problem when you construct more than one instance  
}  
  
  
bool FrameGrabber::Initialize(const std::string& config)  
{  
    assert(videoSource);  
    if (FAILED(videoSource->BuildGraphFromXMLString(const_cast<char*>(config.c_str()))))  
    {  
        printf("[ERROR] could NOT build filtergraph\n");  
        return false;  
    }  
  
    getMediaFormatValues();  
  
    return true;  
}  
  
  
void FrameGrabber::getMediaFormatValues()  
{  
    //get current values  
    LONG        w = 0;  
    LONG        h = 0;  
    double      f = 0.0;  
    PIXELFORMAT pxf;  
  
    assert(videoSource);  
    if (FAILED(videoSource->GetCurrentMediaFormat(&w, &h, &f, &pxf)))  
    {  
        printf("[ERROR] could NOT retrieve media format\n");  
        return;  
    }  
  
    //assign values back to class members  
    width    = w;  
    height   = h;  
    fps      = f;  
    format   = pxf;  
    printf("[INFO] chosen mediaformat: w=%d  h=%d  fps=%.f  format=%d\n", w, h, f, pxf);  
  
    numBytes = 1;   //choose grayscale as default  
    //print info and set number of bytes (components, channels)  
    switch (pxf)  
    {  
//--- only handle the two most important cases here!! ---  
        case PIXELFORMAT_RGB24:  
            numBytes = 3;  
            printf("[INFO] current pixel format is RGB24\n");  
            break;  
  
        case PIXELFORMAT_RGB32:  
            numBytes = 4;  
            printf("[INFO] current pixel format is RGB32\n");  
            break;  
//--------------------------------------  
  
        default:  
            printf("[WARNING] something is wrong with current pixel format\n");  
            break;  
    }  
}  
  
  
bool FrameGrabber::Start()  
{  
    if (!isInitialized)  
    {  
        printf("[ERROR] could NOT start streaming, NOT initialized yet\n");  
        return false;  
    }  
  
    //TODO: replace this naive checks  
    assert(format != PIXELFORMAT_UNKNOWN);  
    assert(format != PIXELFORMAT_INVALID);  
    assert(numBytes > 0);  
    assert(width    > 0);  
    assert(height   > 0);  
    assert(fps      > 0);  
    assert(videoSource);  
  
    printf("[INFO] trying to enable sample memory buffer and start querying threads\n");  
    if (FAILED(videoSource->EnableMemoryBuffer(3))) //here 3 concurrent threads will concurrently query for samples  
    {  
        printf("[ERROR] could NOT start streaming\n");  
        return false;  
    }  
  
    printf("[INFO] trying to start streaming\n");  
    if (FAILED(videoSource->Run()))  
    {  
        printf("[ERROR] could NOT start streaming\n");  
        return false;  
    }  
  
    return true;  
}  
  
  
void FrameGrabber::Stop(bool forceToStop)  
{  
    printf("[INFO] trying to stop streaming\n");  
    assert(videoSource);  
    if (videoSource)  
        videoSource->Stop(forceToStop);  
}  
  
  
void FrameGrabber::GetCurrentFrameData(unsigned char* dst)  
{  
    assert(videoSource);  
    assert(videoSource->IsGraphInitialized());  
    assert(dst);  
    if (!videoSource  ||  !videoSource->IsGraphInitialized()  ||  !dst)  
    {  
        printf("[WARNING] can NOT grab current frame\n");  
        return;  
    }  
  
    //taken from reactivision source (dslibcamera.cpp), magic values?, better strategy?  
    DWORD wait_result = videoSource->WaitForNextSample(/*INFINITE*/ (long)(100.f / fps));  
    /*if (wait_result != WAIT_OBJECT_0)  
    {  
        //wait another 1.5 seconds  
        wait_result = videoSource->WaitForNextSample(1500);  
        if (wait_result != WAIT_OBJECT_0)  
        {  
            //printf("[WARNING] camera stopped\n");  
            //Stop();  
        }  
    }*/  
  
    //if (wait_result == WAIT_OBJECT_0)  
    {  
        if (SUCCEEDED(videoSource->CheckoutMemoryBuffer(&mbHandle, &buffer)))  
        {  
            timeStamp = videoSource->GetCurrentTimestamp();  
            //printf("LIVE VIDEO TIMESTAMP: %d\n", timeStamp);  
  
            //prepare array  
            ::memcpy(dst, buffer, width*height*numBytes);    //3 works with RGB24  
  
            videoSource->CheckinMemoryBuffer(mbHandle);  
            buffer = 0;  
        }  
        else  
        {  
            printf("[WARNING] could NOT checkout membuffer (get sample)\n");  
        }  
    }   
    /*else  
    {  
        printf("[WARNING] could NOT aquire camera input\n");  
    }*/  
}  
  
  
uint FrameGrabber::GetNumberOf8BitChannels() const  
{  
    return numBytes;  
}  
  
  
uint FrameGrabber::GetWidth() const  
{  
    return width;  
}  
  
  
uint FrameGrabber::GetHeight() const  
{  
    return height;  
}  
  
  
void FrameGrabber::ShowSettings() const  
{  
    assert(videoSource);  
    if (videoSource &&  videoSource->IsGraphInitialized())  
        videoSource->ShowFilterProperties();  
    else  
        printf("[ERROR] can NOT show settings, NOT initialized yet\n");  
}  
  
  
int FrameGrabber::HasDroppedFrames() const  
{  
    assert(videoSource);  
  
    int numDroppedFrames = 0;  
    numDroppedFrames = videoSource->HasDroppedFrames();  
  
    return numDroppedFrames;  
}  
  

USAGE:

  
  
you need in your TestApp:  
    FrameGrabber*  framegrabber;  
    colorImage*    video;      //for processing further, similar like openframeworks opencv addon image  
    unsigned char* videoData;  //to read out of capture device  
  
in  
void TestApp::setup()  
{  
    //...  
  
    //debug/test strings, could be external xml file  
    std::string config = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><dsvl_input><camera friendly_name=\"AVT 1394 camera\" show_format_dialog=\"false\"><pixel_format><RGB24 /></pixel_format></camera></dsvl_input>";  
  
    framegrabber = new FrameGrabber(config);  
    assert(framegrabber);  
    if (framegrabber  &&  framegrabber->IsInitialized())  
    {  
        //framegrabber->ShowSettings(); //to force to show settings  
        framegrabber->Start();  
  
        int numChannels = framegrabber->GetNumberOf8BitChannels();   //NOTE: it is very important for the array (char) that it is 8 bit!  
        if (numChannels == 3)   //RGB (RGB24)  
        {  
            video = new colorImage(camWidth, camHeight, false, false);  
            assert(video);  
  
            videoData = new unsigned char[camWidth*camHeight*numChannels];  
            assert(videoData);  
            ZERO_MEMORY(videoData, camWidth*camHeight*numChannels);  
        }  
        else  
        if (numChannels == 4)   //RGBA  
        {  
            printf("[ERROR] RGBA format NOT implemented yet\n");  
        }  
        else  
        if (numChannels == 1)   //GRAYSCALE, default return value  
        {  
            printf("[ERROR] grayscale format NOT implemented yet\n");  
        }  
        else                    //0, ctor default  
        {  
            printf("[ERROR] invalid return value for media format / number of channels (%d)\n", numChannels);  
        }  
    }  
    else  
    {  
        printf("[FATAL] could NOT allocate or initialize framegrabber\n");  
    }  
}  
  
  
in  
void TestApp::update()  
{  
    assert(framegrabber);  
    if (framegrabber)  
    {  
        assert(videoData);  
        framegrabber->GetCurrentFrameData(videoData);  
  
        assert(video);  
        if (video)  
        {  
            video->setFromNonIppImage(videoData, camWidth, camHeight, camWidth*framegrabber->GetNumberOf8BitChannels());  
            //video is not ready for drawing!  
        }  
    }  
}  
  

i hope this helps! tell me if you need the changed DSVL sourcecode (a pity that this software is not mainteined anymore).
cheers, didi

didito, I created a addon for DSVL that seems to work ok and is similar to yours, but a little less complicated/feature rich. One major problem though, occasionally it slows down randomly to low fps. I’m not sure what the hangup is. Did you experience anything similar and is this what you had to customize?

This is my class:

  
#include "ofxDSVL.h"  
  
ofxDSVL::ofxDSVL()  
{  
	//touchlib  
	dsvl_vs = NULL;  
    buffer = NULL;  
	pbuffer = NULL;  
  
    g_Timestamp = _INVALID_TIMESTAMP;  
  
    cap_width = 0;  
    cap_height = 0;  
    cap_fps = 0.0;  
  
    colour = false;  
    newFrame = false;  
  
    bytes = 0;  
}  
  
void ofxDSVL::setDeviceID(int id)  
{  
}  
  
unsigned char * ofxDSVL::getPixels()  
{  
	return pbuffer;  
}  
  
int ofxDSVL::getDeviceID()  
{  
    return 0;  
}  
  
void ofxDSVL::initDSVL()  
{  
    CoInitialize(NULL);  
    dsvl_vs = new DSVL_VideoSource();  
  
    if(FAILED(dsvl_vs->BuildGraphFromXMLFile("DSVL_config.xml"))){return;}  
  
	if(FAILED(dsvl_vs->GetCurrentMediaFormat(&cap_width,&cap_height,&cap_fps,NULL))){return;}  
  
    PIXELFORMAT pxf;  
    if(FAILED(dsvl_vs->GetCurrentMediaFormat(&cap_width,&cap_height,&cap_fps,&pxf))){return;}  
  
	//choose byte size. grayscale by default  
	bytes = 1;     
    switch (pxf)  
    {	//only handle the two most important cases here  
        case PIXELFORMAT_RGB24:  
            bytes = 3;  
            printf("[INFO] current pixel format is RGB24\n");  
            break;  
  
        case PIXELFORMAT_RGB32:  
            bytes = 4;  
            printf("[INFO] current pixel format is RGB32\n");  
            break;  
	}  
  
    if(FAILED(dsvl_vs->EnableMemoryBuffer(3))){return;}  
    // three concurrent threads will concurrently query for samples  
	if(FAILED(dsvl_vs->Run())){return;}  
  
	camWidth  = cap_width;  
	camHeight = cap_height;   
	fps		  = (int)cap_fps;  
  
	pbuffer = new unsigned char[camWidth*camHeight*bytes];  
}  
  
bool ofxDSVL::isFrameNew()  
{  
	//taken from reactivision source (dslibcamera.cpp), magic values?, better strategy?  
	DWORD wait_result = dsvl_vs->WaitForNextSample(/*INFINITE*/ (long)(1000.0f/fps));  
	{  
		if (SUCCEEDED(dsvl_vs->CheckoutMemoryBuffer(&g_mbHandle, &buffer)))  
		{  
			g_Timestamp = dsvl_vs->GetCurrentTimestamp();  
			//prepare array  
			memcpy(pbuffer, buffer, camWidth*camHeight*bytes);  
			dsvl_vs->CheckinMemoryBuffer(g_mbHandle);  
			buffer = 0;  
  
			return true;  
		}  
		else  
		{  
			printf("[WARNING] could NOT checkout membuffer (get sample)\n");  
		}  
	}  
  
	return false;  
}  
  
int ofxDSVL::getCamWidth()  
{  
    return camWidth;  
}  
  
int ofxDSVL::getCamHeight()  
{  
    return camHeight;  
}  
  
int ofxDSVL::getNumByes()  
{  
	return bytes;  
}  
  
void ofxDSVL::showSettingsDialog() {  
  
	dsvl_vs->ShowFilterProperties();  
}  
  
//Clean up  
ofxDSVL::~ofxDSVL()  
{  
    dsvl_vs->Stop();  
    dsvl_vs->ReleaseGraph();  
	if (pbuffer!=NULL) delete []pbuffer;  
    delete dsvl_vs;  
}