sending images over network sendRawBytes

[diediaga]

Hi
I just try to change a bit the TCP/IP addon example of version 0,05 (linux codeblocks). the idea is:
- a client with a camera connected send a still image to the server every 5 seconds.
- a server that receive and display this image.

for sending (the client) I tried with this code:

Code:

            int totalPixels = camWidth*camHeight*3;
            unsigned char * pixels = vidGrabber.getPixels();
            tcpClient.sendRawBytes((const char*)pixels, totalPixels);
            mono.drawString("image sent!!", 15, 100);

for receiving and displaying (the server) I tried with this code:

Code:

        int totalPixels = camWidth*camHeight*3;
        char * pixels = NULL;
        if(TCP.receiveRawBytes(i, pixels, totalPixels==TRUE)){
            monosm.drawString("image  successfully received", 15, 100);
            videoTexture.loadData((unsigned char *)pixels, camWidth,camHeight, GL_RGB);
            nImagen++;
        }
	videoTexture.draw(20,120);

but it is not working. I cant see the picture... but I can see the texts. (image sent and received).
please help

thanks
diego[/code]

[arturo]

don't know if it's a typo but in:

Code:

if(TCP.receiveRawBytes(i, pixels, totalPixels==TRUE)){ 

you should have:

Code:

if(TCP.receiveRawBytes(i, pixels, totalPixels)==TRUE){ 

note the last parentesis... If not you're only reciving one pixel

[diediaga]

Hi Arturo, thanks for your replay.
I try your suggestions but and I still have the same problem.
I paste here the full code just in case you someone can help.

CODE FOR SERVER

Code:

#include "testApp.h"
#include "stdio.h"


//--------------------------------------------------------------
void testApp::setup(){
	//we run at 60 fps!
	ofSetVerticalSync(true);

	//load our type
	mono.loadFont("type/mono.ttf", 9);
	monosm.loadFont("type/mono.ttf", 8);

	//setup the server to listen on 11999
	TCP.setup(11999);

    camWidth 		= 320;	// try to grab at this size.
	camHeight 		= 240;

	videoTexture.allocate(camWidth,camHeight, GL_RGB);
	nImagen = 0;

}

//--------------------------------------------------------------
void testApp::update(){
	ofBackground(20, 20, 20);

	//for each client lets send them a message letting them know what port they are connected on
	for(int i = 0; i < TCP.getNumClients(); i++){
		TCP.send(i, "hello client - you are connected on port - "+ofToString(TCP.getClientPort(i)) );
	}



}

//--------------------------------------------------------------
void testApp::draw(){

	ofSetColor(0xDDDDDD);
	mono.drawString("TCP SERVER Example \n\nconnect on port: "+ofToString(TCP.getPort()), 10, 20);

	ofSetColor(0x000000);
	ofRect(10, 60, ofGetWidth()-24, ofGetHeight() - 65 - 15);

	ofSetColor(0xDDDDDD);

	//for each connected client lets get the data being sent and lets print it to the screen
	for(int i = 0; i < TCP.getNumClients(); i++){

		//give each client its own color
		ofSetColor(255 - i*30, 255 - i * 20, 100 + i*40);

		//calculate where to draw the text
		int xPos = 15;
		int yPos = 80 + (12 * i * 4);

		//get the ip and port of the client
		string port = ofToString( TCP.getClientPort(i) );
		string ip   = TCP.getClientIP(i);
		string info = "client "+ofToString(i)+" -connected from "+ip+" on port: "+port;


		//if we don't have a string allocated yet
		//lets create one
		if(i >= storeText.size() ){
			storeText.push_back( string() );
		}

		//we only want to update the text we have recieved there is data
		string str = TCP.receive(i);


		if(str.length() > 0){
			storeText[i] = str;
		}

		//draw the info text and the received text bellow it
		monosm.drawString(info, xPos, yPos);
		monosm.drawString(storeText[i], 25, yPos + 20);

        int totalPixels = camWidth*camHeight*3;
        char * pixels = NULL;

//FOR RECEIVE THE IMAGE IN RAWBYTES!!! DONT WORK!!
        if (TCP.receiveRawBytes(i, pixels, totalPixels)==TRUE){
            monosm.drawString("imagen recibida: "+ ofToString(nImagen), 15, 110);
            videoTexture.loadData((unsigned char *)pixels, camWidth,camHeight, GL_RGB);
            nImagen++;
        }


    }
    videoTexture.draw(20,120);



}

//--------------------------------------------------------------
void testApp::keyPressed(int key){
}

//--------------------------------------------------------------
void testApp::keyReleased(int key){
}


//--------------------------------------------------------------
void testApp::mouseMoved(int x, int y ){
}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int button){

}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int button){

}

//--------------------------------------------------------------
void testApp::mouseReleased(){
}

CODE FOR CLIENT

Code:

#include "testApp.h"

#define RECONNECT_TIME 400

//--------------------------------------------------------------
void testApp::setup(){

    // we don't want to be running to fast
	ofSetVerticalSync(true);

	//load our type
	mono.loadFont("type/mono.ttf",9);
	monosm.loadFont("type/mono.ttf",8);

	//some variables

	//have we typed
	typed	= false;

	//our typing position
	pos		= 0;

	//our send and recieve strings
	msgTx	= "";
	msgRx	= "";
	bb = "";

	//are we connected to the server - if this fails we
	//will check every few seconds to see if the server exists
	weConnected = tcpClient.setup("127.0.0.1", 11999);

	connectTime = 0;
	deltaTime = 0;

	tcpClient.setVerbose(true);

    camWidth 		= 320;	// try to grab at this size.
	camHeight 		= 240;

	vidGrabber.setVerbose(true);
	vidGrabber.initGrabber(camWidth,camHeight);

	videoInverted 	= new unsigned char[camWidth*camHeight*3];
	videoTexture.allocate(camWidth,camHeight, GL_RGB);

    enviar = 0;

}

//--------------------------------------------------------------
void testApp::update(){
	ofBackground(100,100,100);

	vidGrabber.grabFrame();

    int totalPixels = camWidth*camHeight*3;

		//we are connected - lets send our text and check what we get back
	if(weConnected){
		tcpClient.send(msgTx);

		//if data has been sent lets update our text
		string str = tcpClient.receive();
		if( str.length() > 0 ){
			msgRx = str;
		}

		if (enviar > 10){ //SEND AN IMAGE EVERY 10 LOOPS

            unsigned char * pixels = vidGrabber.getPixels();

            if(tcpClient.sendRawBytes((const char*)pixels, totalPixels)==TRUE){
            videoTexture.loadData(pixels, camWidth,camHeight, GL_RGB);
            enviar = 0;
            }
		}
		enviar++;



	}else{
		//if we are not connected lets try and reconnect every 5 seconds
		deltaTime = ofGetElapsedTimeMillis() - connectTime;

		if( deltaTime > 5000 ){
			weConnected = tcpClient.setup("127.0.0.1", 11999);
			connectTime = ofGetElapsedTimeMillis();
		}

	}
}

//--------------------------------------------------------------
void testApp::draw(){

    ofSetColor(0xffffff);

	vidGrabber.draw(20,350);
	videoTexture.draw(20+camWidth,350,camWidth,camHeight);

	mono.drawString("openFrameworks TCP Send Example", 15, 30);

	if(typed){
		monosm.drawString("sending:", 15, 55);
		monosm.drawString(msgTx, 85, 55);
	}
	else{
		if(weConnected)monosm.drawString("status: type something to send data to port 11999", 15, 55);
		else monosm.drawString("status: server not found. launch server app and check ports!\n\nreconnecting in "+ofToString( (5000 - deltaTime) / 1000 )+" seconds", 15, 55);
	}

	monosm.drawString("from server: \n"+msgRx, 15, 270);

	monosm.drawString("enviar ="+ofToString(enviar),15,300);


}

//--------------------------------------------------------------
void testApp::keyPressed  (int key){

    	//you can only type if you're connected
	if(weConnected){


		if(key == 13)key = '\n';
		if(key == 8 || key == 127){
			if( pos != 0 ){pos--;
				msgTx = msgTx.substr(0,pos);
			}else msgTx = "";
		}else{
			msgTx.append(1, (char) key);
			pos++;
		}
		typed = true;


	}


	// in fullscreen mode, on a pc at least, the
	// first time video settings the come up
	// they come up *under* the fullscreen window
	// use alt-tab to navigate to the settings
	// window. we are working on a fix for this...

	//if (key == 's' || key == 'S'){
	//	vidGrabber.videoSettings();
	//}

}

//--------------------------------------------------------------
void testApp::keyReleased  (int key){

}

//--------------------------------------------------------------
void testApp::mouseMoved(int x, int y ){
}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int button){
}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int button){
}

//--------------------------------------------------------------
void testApp::mouseReleased(){

}

[zach]

three things:

a) I think you should remove the message text part from both client and server

b) check the return types --- receiveRawBytes doesn't return bool, it reaturns the number of bytes actually read.  I suspect that you are not asking the questions right here:

Code:

if (TCP.receiveRawBytes(i, pixels, totalPixels)==TRUE)

and that you if you actually printed out how many bytes you did read, you might see something intersting (like, it didn't read enough, etc)

c) I'm just guessing but I think the amount of data might be really too high:

320*240*3 = 230400 bytes = 230kb

modern connnections are 100 megabit or 10 megabit,
10 megabit = approx 1 megabyte per second / 3 = 300kb

I'd start with something smaller in binary --- 10 bytes, make sure they are working then move your way up to image data.  

can you try coding a binary example very simple first?


hope that helps!
zach

[zach]

ps move to extend since this features an addon (network)

[diediaga]

Hi Zach

I am sending a very small package 10bytes
I tried with

Code:

        
int bitesRecibidos = TCP.receiveRawBytes(i, pixels, totalPixels);
monosm.drawString("bites recibidos: "+ ofToString(bitesRecibidos), 15, 110);

but it always print -1

do you have any idea?

[zach]

can you zip sender and client src and data folders?

thanks!
zach

[diediaga]

Sure!
the links are.

http://www.lalalab.org/tmp/src_client.zip
http://www.lalalab.org/tmp/data_client.zip

http://www.lalalab.org/tmp/src_server.zip
http://www.lalalab.org/tmp/data_server.zip

thanks very much for your help.-

diego

[zach]

I'm working on this, but it's not there yet.  I'm able to pass data, but after time it gets offset, so there are issues.  We need to flush the buffer on incomplete reads (since sometimes we can't read the whole amount of binary data) we can get data across now, but it's not working out yet.



I think it's going to be much better to do this as UDP - I'll try to code up an example.  I have some ideas about how to solve the offset too so will keep hacking.

(note, it's better now that I am flushing the buffer after incomplete reads, but it's not perfect - there are still small issues -- I think the flush has to read a full image worth of data to truly flush.  )

take care,
zach

[arturo]

Hi

Not sure if this is the problem, but when you send big amounts of data, the best solution is to use a terminator and not expect that all the data arrives in on read but continue reading since you got the terminator.

You can have the problem of having the terminator in the middle of your data, one solution is to use something very unprobable to appear, or to be completely sure, duplicate the false terminators and when receiving, you know that double terminators must be converted to only one and that the real one is that, that appears only one time.

[zach]

I agree that a terminator would be good, but you should be able to do without a termintor if the data is always the same size.

I had really good luck just now w/ writing a better flush.

I think the thing is:

a) when you have alot of data, you have to deal with partial reads.
b) likely this ought to be implemented in the higher level object, that it waits until it reads x amount of bytes before returning.  currently the higher level object wraps the lower level obect that just tries to read and returns partial reads.
c) adding a flush which is dependent on the remaining bytes left from partial reads solves the problem, but means you loose that frame of data.
d) also, flush (c) uses a while loop, which makes me scared of getting stuck forever.


this is an example of flushing based on the remaining amount of bytes:

Code:

if (bitesRecibidos == totalPixels){
    videoTexture.loadData(pixels, camWidth, camHeight, GL_RGB);
} else if (bitesRecibidos > 0) {
        int remainderCount = totalPixels - bitesRecibidos;
        while (remainderCount > 0){
            bitesRecibidos = TCP.receiveRawBytes(i, (char *)pixels, 1);
            if (bitesRecibidos == 1) remainderCount--;
        }
}

this while loop is a bit frightening to me.

I'll post everything shortly once it's clean, but it's been working really well with little delay.  The throttling is handled on the server side (not the client side as diego has posted) and seems snappy.



more shortly !

take care,
zach

[arturo]

yes, didn't realize that you already know the size of the data.

About the while, yes it's not the best solution but as long as you don't have non-blocking receiving through a callback, i think is the only one.

Also you can increase the tcp buffer size for that socket setting a socket property:

Code:

int size=... //size in bytes
setsockopt(m_hSocket, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size));

for the receiver and

SO_SNDBUF

for the sender

[joshua noble]

I know this an old post but I was just curious if there were updates to this. I'm working on the same thing and have independently come to the same conclusions. My results though aren't really working. I'm thinking the problem is here:

if (received == totalPixels){
            text.loadData(messageBytes, 320, 240, GL_RGB);
            received = 0;
            newDraw = true;
         } else {
            received = tcpClient.receiveRawBytes((char *) messageBytes, 1);
            if (received > 0) {
               int toGo = totalPixels - bytesReceived;
               while (toGo > 0){
                  tcpClient.receiveRawBytes((char *) messageBytes[bytesReceived], 1);
                  received++;
                  toGo--;
               }
               printf(" %i ", received);
            }
         }

I'm thinking that the while loop isn't grabbing all the pixels (even though it has the right number of loop throughs? It should grab 1 char from the buffer each time, if I'm not mistaken. I just coded all this stuff up and found it not working, so I figured I'd troll around here and see what I could find. This looks wrong, but I'm not sure how the receiveRawBytes is supposed to work with really large data like this.

tcpClient.receiveRawBytes((char *) messageBytes[bytesReceived], 1);

Thanks in advance for any advice.

edit ------

changed to this, seems better:

   tcpClient.receiveRawBytes(recByte, 1);
   memcpy(&messageBytes[bytesReceived], recByte, 1);

still no magic tho.

[theo]

Hi Josh

Do you think you could post up your examples so I can help debug the issue?
One other option that might help is to switch the tcp mode to blocking and put it in a thread - but I agree it should also work the way you are doing it as well.

Theo

edit - also make sure you have the latest ofxNetwork
http://svn.openframeworks.cc/browser/li ... twork_src_

and the latest ofxThread
http://svn.openframeworks.cc/browser/li ... hread_src_

[joshua noble]

Here's my Client:

Code:

void NetworkClient::setup(){	 

	// we don't want to be running to fast
	ofSetVerticalSync(true);

	connection = tcpClient.setup("127.0.0.1", 11999);
	
	connectTime = 0;	
	deltaTime = 0;
	
	tcpClient.setVerbose(true);
	received = 0;
	text.allocate(320, 240, GL_RGB);
	totalPixels = 320 * 240 * 3;
}

void NetworkClient::update(){
	ofBackground(255, 255, 255);
	if(connection){
		
			int i = 0;
			if (received == totalPixels){
				text.loadData(messageBytes, 320, 240, GL_RGB);
				img.setFromPixels(messageBytes, 320, 240, OF_IMAGE_COLOR, true);
				received = 0;
				newDraw = true;
			} else {
				received = tcpClient.receiveRawBytes((char *) messageBytes, 1);
				char * recByte;
				if (received > 0) {
					int remainderCount = totalPixels - received;
					while (remainderCount > 0){
						tcpClient.receiveRawBytes(recByte, 1);
						memcpy(&messageBytes[received], recByte, 1); // this is kinda dumb, I know
						received++;
						remainderCount--;
					}
					printf(" %i ", received); //prints 230400 i.e. 320*240*3
				}
			} 

	}else{		
		//if we are not connected lets try and reconnect every 5 seconds
		deltaTime = ofGetElapsedTimeMillis() - connectTime;
		
		if( deltaTime > 5000 ){
			connection = tcpClient.setup("127.0.0.1", 11999);
			connectTime = ofGetElapsedTimeMillis();
		}
	}
}

void NetworkClient::draw(){
	ofSetColor(0, 0, 0);
	text.draw(0, 0, 320, 240);
}

and here's the server

Code:

#include "NetworkServer.h"
#include "stdio.h"



void NetworkServer::setup(){	 
	//we run at 60 fps!
	ofSetVerticalSync(true);

	//setup the server to listen on 11999
	TCP.setup(11999);
	
	vidHeight = 240;
	vidWidth = 320;
	
	video.setVerbose(true);
	video.initGrabber(320, 240);
	sendFrame = false;
}


void NetworkServer::update() {
	ofBackground(20, 20, 20);
	
	video.grabFrame();

	if(sendFrame) {
		
		// I should probably be throttling this here, sending, say 2048 bytes at a time I guess
		unsigned char * pixels = video.getPixels(); 
		//I could loop through and cast this, dynamic_cast it, or...well, should be fine
		TCP.sendRawBytesToAll(pixels, 320*240*3);
		sendFrame = false;
	}
}


void NetworkServer::draw(){

	ofSetColor(0xffffff);
	video.draw(0, 0);
}


void NetworkServer::keyPressed(int key){
	sendFrame = true;
}

I'm think the problem is just that I'm not throttling on the server side. I really should be sending packets of 2048 bytes and I'll change this to do that later, but I'm wondering if there's something else I've gotten wrong. You're right, setting the port to non-blocking and using a thread is the right way to go for something series, but that's a little too complex for my purposes (i.e. for the example code, I don't want to get into that too much with readers) Thanks Theo![/code]