Convert dual fisheye video to equirectangular with

I used open framework to convert dual fisheye video (received from ricoh theta S camera) to equirectangular format (for an iOS application).

dual fisheye image sample:

equirectangular image sample:
https://developers.theta360.com/intl/common/img/equirectangular.jpg

I use below shader:

equirectangular.frag

// based on ThetaShaderPack_20150926 (http://stereoarts.jp/) written by Nora.
#ifdef GL_ES
// define default precision for float, vec, mat.
precision highp float;
#endif

#define PI 3.14159265358979
#define _THETA_S_Y_SCALE	(640.0 / 720.0)

uniform sampler2D mainTex;
uniform float radius;
uniform vec4 uvOffset;

varying vec2 texCoordVarying;

void main (void) {
    vec2 revUV = texCoordVarying.st;
    if (texCoordVarying.x <= 0.5) {
        revUV.x = revUV.x * 2.0;
    } else {
        revUV.x = (revUV.x - 0.5) * 2.0;
    }
    
    revUV *= PI;

    vec3 p = vec3(cos(revUV.x), cos(revUV.y), sin(revUV.x));
    p.xz *= sqrt(1.0 - p.y * p.y);

    float r = 1.0 - asin(p.z) / (PI / 2.0);
    vec2 st = vec2(p.y, p.x);

    st *= r / sqrt(1.0 - p.z * p.z);
    st *= radius;
    st += 0.5;
    
    if (texCoordVarying.x <= 0.5) {
        st.x *= 0.5;
        st.x += 0.5;
        st.y = 1.0 - st.y;
        st.xy += uvOffset.wz;
    } else {
        st.x = 1.0 - st.x;
        st.x *= 0.5;
        st.xy += uvOffset.yx;
    }
    
    st.y = st.y * _THETA_S_Y_SCALE;
    
    gl_FragColor = texture2D(mainTex, st);
}

equirectanguler.vert

uniform mat4 projectionMatrix;
uniform mat4 modelViewMatrix;
uniform mat4 textureMatrix;
uniform mat4 modelViewProjectionMatrix;

attribute vec4  position;
attribute vec4  color;
attribute vec3  normal;
attribute vec2  texcoord;

varying vec2 texCoordVarying;

void main() {
    texCoordVarying = texcoord;
    gl_Position = modelViewProjectionMatrix * position;
}

with below code:
main.mm

#include "ofApp.h"

int main() {
    
    //  here are the most commonly used iOS window settings.
    //------------------------------------------------------
    ofiOSWindowSettings settings;
    settings.enableRetina = false; // enables retina resolution if the device supports it.
    settings.enableDepth = false; // enables depth buffer for 3d drawing.
    settings.enableAntiAliasing = false; // enables anti-aliasing which smooths out graphics on the screen.
    settings.numOfAntiAliasingSamples = 0; // number of samples used for anti-aliasing.
    settings.enableHardwareOrientation = false; // enables native view orientation.
    settings.enableHardwareOrientationAnimation = false; // enables native orientation changes to be animated.
    settings.glesVersion = OFXIOS_RENDERER_ES2; // type of renderer to use, ES1, ES2, ES3
    settings.windowMode = OF_FULLSCREEN;
    ofCreateWindow(settings);
    
	return ofRunApp(new ofApp);
}

offApp.mm

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){	
    ofDisableArbTex();
    
    devices = theta.listDevices();
    bool isDeviceConnected = false;
    for(int i = 0; i < devices.size(); i++){
        if(devices[i].deviceName == "RICOH THETA S"){
            theta.setDeviceID(devices[i].id);
            isDeviceConnected = true;
        }
    }
    if(!isDeviceConnected){
        ofLog(OF_LOG_ERROR, "RICOH THETA S is not found.");
    }
    theta.initGrabber(360, 568);
    
    shader.load("shaders/equirectanguler");
    
    fbo.allocate(320, 568);
    
    sphere = ofSpherePrimitive(568, 64).getMesh();
    for(int i=0;i<sphere.getNumTexCoords();i++){
        sphere.setTexCoord(i, ofVec2f(1.0) - sphere.getTexCoord(i));
    }
    for(int i=0;i<sphere.getNumNormals();i++){
        sphere.setNormal(i, sphere.getNormal(i) * ofVec3f(-1));
    }
    
    offset.set("uvOffset", ofVec4f(0,0.0,0,0.0), ofVec4f(-0.1), ofVec4f(0.1));
    radius.set("radius", 0.445, 0.0, 1.0);
    showSphere.set("showSphere", false);
    thetaParams.add(offset);
    thetaParams.add(radius);
    gui.setup(thetaParams);
    gui.add(showSphere);
    
    cam.setAutoDistance(false);
    cam.setDistance(0);
}

//--------------------------------------------------------------
void ofApp::update(){
    theta.update();
}

//--------------------------------------------------------------
void ofApp::draw(){
    if(theta.isFrameNew()){
        
        fbo.begin();
        ofClear(0);
        shader.begin();
        shader.setUniformTexture("mainTex", theta.getTexture(), 0);
        shader.setUniforms(thetaParams);
        theta.draw(0, 0, 320, 568);
        shader.end();
        fbo.end();
        
    }
    
    if(!showSphere){
        
        fbo.draw(0, 0, 320, 568);
        
    }else{
        
        ofEnableDepthTest();
        cam.begin();
        fbo.getTexture().bind();
        sphere.draw();
        fbo.getTexture().unbind();
        cam.end();
        
    }
    
    ofDisableDepthTest();
    gui.draw();
}

//--------------------------------------------------------------
void ofApp::exit(){

}

//--------------------------------------------------------------
void ofApp::touchDown(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchMoved(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchUp(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchDoubleTap(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchCancelled(ofTouchEventArgs & touch){
    
}

//--------------------------------------------------------------
void ofApp::lostFocus(){

}

//--------------------------------------------------------------
void ofApp::gotFocus(){

}

//--------------------------------------------------------------
void ofApp::gotMemoryWarning(){

}

//--------------------------------------------------------------
void ofApp::deviceOrientationChanged(int newOrientation){

}

offApp.h

#pragma once

#include "ofxiOS.h"
#include "ofxGui.h"

class ofApp : public ofxiOSApp {
	
    public:
        void setup();
        void update();
        void draw();
        void exit();
	
        void touchDown(ofTouchEventArgs & touch);
        void touchMoved(ofTouchEventArgs & touch);
        void touchUp(ofTouchEventArgs & touch);
        void touchDoubleTap(ofTouchEventArgs & touch);
        void touchCancelled(ofTouchEventArgs & touch);

        void lostFocus();
        void gotFocus();
        void gotMemoryWarning();
        void deviceOrientationChanged(int newOrientation);

    ofVideoGrabber theta;
    vector<ofVideoDevice> devices;
    ofShader shader;
    ofFbo fbo;
    ofEasyCam cam;
    ofVboMesh sphere;
    
    ofParameter<ofVec4f> offset;
    ofParameter<float> radius;
    ofParameter<bool> showSphere;
    ofParameterGroup thetaParams;
    ofxPanel gui;
};

And here is the result:
on iphone5s
https://i.gyazo.com/c032300157ba0ca15661825e8094ceff.png

pls point me out what i missed.

1 Like

Do you still have this problem? Are you using sample from this:
https://github.com/yasuhirohoshino/thetaRealtimeEquirectangular?

I am still a beginner in openframework but I have used the sample and work perfectly for me in Windows. The codes are exactly same with me with the only differences are in main.cpp:
//========================================================================
#include “ofMain.h”
#include “ofApp.h”
int main( ){
ofGLWindowSettings settings;
settings.setGLVersion(4, 0);
settings.width = 1280;
settings.height = 640;
ofCreateWindow(settings);
// this kicks off the running of my app
// can be OF_WINDOW or OF_FULLSCREEN
// pass in width and height too:
ofRunApp(new ofApp());
}

May be the opengl version??

Thank you.
The code seem works in OSX but it does not work with iOS device.
So i have to convert that code to run in iOS device. But i failed. So finally i decided to switch to GPUImage framework.

Its nice to hear that. Anyway, if you only want to get the Equirectangular format of Ricoh Theta streaming you can just read the “THETA UVC Blender” rather than “RICOH THETA S,” It will give you equirectangular output directly, not the dual sphere…

Good luck for your work!