Multiple FFTs/RMS

Hi All,

I’ve recently picked up an old project where I was attempting to to compute audio values from nine audio files that are playing at the same time and then mixed down to stereo for output.

I can’t seem to find a way to compute each track before the mix, in order to send the values out to light some LEDs attached to an Arduino, communicating via firmata.

I am confident that the communication between the Arduino and computer (RPi in this case) is working properly. But am stuck on a way of getting the lights to react to each separate sound.

Here is my ofApp.cpp

#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){
    
//    potValue = "analog pin:";
//    str = " ";

    // replace the string below with the serial port for your Arduino board
    // you can get this from the Arduino application or via command line
    // for OSX, in your terminal type "ls /dev/tty.*" to get a list of serial devices
	ard.connect("/dev/tty.usbmodem145301", 57600);
	
	// listen for EInitialized notification. this indicates that
	// the arduino is ready to receive commands and it is safe to
	// call setupArduino()
	ofAddListener(ard.EInitialized, this, &ofApp::setupArduino);
	bSetupArduino	= false;	// flag so we setup arduino when its ready, you don't need to touch this :)
    
    //phase = 0;
    //ofSoundStreamSetup(9, 0); // 2 output channels (stereo), 0 input channels

    sound.resize(faces);
    //soundSpectrum.resize(faces);
    
    for (int i = 0; i < faces; i++){
        sound[i].load((ofToString(i))+".wav");
        sound[i].setLoop(false);
        sound[i].setVolume(0.0);
        sound[i].play();
        //sound[i].get
        csoSanity[i] = false;
    }
    
    fft = new float [8];
    for (int i = 0; i < 8; i++)  {
        fft[i] = 0;
    }
    
    bands = 4;

    sound[0].setLoop(true);
}

//--------------------------------------------------------------
void ofApp::update(){

	updateArduino();
    ofSoundUpdate();

    float pos = sound[0].getPosition();
    
    for (int i = 1; i < faces; i++){
        if (sound[0].getPosition() <= 0.009){
            sound[i].setPosition(pos);
            sound[i].play();
        }
    }
    
    soundSpectrum = ofSoundGetSpectrum(bands);
    //float * value = ofSoundGetSpectrum(bands);
    for (int i = 0; i < bands; i++) {
        fft[i] *= 0.8f;
        if (fft[i] < soundSpectrum[i])    {
           fft[i] = soundSpectrum[i];
        }
    }
//
//    for (int i = 0; i < bands; i++) {
//        unNormal[i] = ofMap(fftSmooth[i], 0, 0.1f, 0, 255);
//    }
}

//--------------------------------------------------------------
void ofApp::setupArduino(const int & version) {
	
	// remove listener because we don't need it anymore
	ofRemoveListener(ard.EInitialized, this, &ofApp::setupArduino);
    
    // it is now safe to send commands to the Arduino
    bSetupArduino = true;
    
    // print firmware name and version to the console
    ofLogNotice() << ard.getFirmwareName(); 
    ofLogNotice() << "firmata v" << ard.getMajorFirmwareVersion() << "." << ard.getMinorFirmwareVersion();
        
    // set pins to analog input & output
    for (int i = 0; i < faces; i++){
        ard.sendAnalogPinReporting((i+2), ARD_ANALOG);
//        ard.sendDigitalPinMode((i), ARD_PWM);
    }
    
    // Listen for changes on the digital and analog pins
//    ofAddListener(ard.EDigitalPinChanged, this, &ofApp::digitalPinChanged);
    ofAddListener(ard.EAnalogPinChanged, this, &ofApp::analogPinChanged);
}

//--------------------------------------------------------------
void ofApp::updateArduino(){

	// update the arduino, get any data or messages.
    // the call to ard.update() is required
	ard.update();

//	// do not send anything until the arduino has been set up
    if (bSetupArduino) {

        makeString();
        

	}
}

// analog pin event handler, called whenever an analog pin value has changed

//--------------------------------------------------------------
void ofApp::analogPinChanged(const int & pinNum) {

    int trigger = 300;
    
    for (int i = 0; i < faces; i++){
        if (ard.getAnalog(i+2) <= trigger && csoSanity[i] != true){
            sound[i].setVolume(0.99);
            counter[i] = ofGetElapsedTimeMillis();
            csoSanity[i] = true;

        } else if (ard.getAnalog(i+2) <= trigger && (ofGetElapsedTimeMillis() - counter[i]) <= 15000 ){
            sound[i].setVolume(0.99);

        } else if (ard.getAnalog(i+2) <= trigger && (ofGetElapsedTimeMillis() - counter[i]) >= 15000 ){
            sound[i].setVolume(0.0);
            csoSanity[i] = true;
        
        } else {
            sound[i].setVolume(0.0);
            csoSanity[i] = false;
        
        }
    }
}
//--------------------------------------------------------------
void ofApp::makeString  (){
//    str = "1,";
    
//    for (int i = 0; i < faces; i++){
//        str += ofToString(i);
//        str += ofToString(sin(ofGetElapsedTimef())) + ",0,0";
////        ard.sendString(str);
//        cout << str << endl;
//    }
}
//--------------------------------------------------------------

//void ofApp::audioOut(float *output, int bufferSize, int nChannels){
//    
//    float pos = sounds[0].position / sounds[0].length;
//    
//    if (pos <= 0.003){
//        for (int i = 1; i < sounds.size(); i++){
//            sounds[i].playOnce();
//            sounds[i].setPosition(pos);
//        }
//    }
//    
//    for (int i = 0; i < bufferSize; i++){
//        
//        double wave = 0.0;
//        for(int i = 0; i < sounds.size(); i++){
//            if(i == 0){
//                waves[i] = sounds[0].play();
//            } else {
//                waves[i] = sounds[i].playOnce();
//            }
//            wave += waves[i];
//        }
//        
//        double crush = decimate.process(wave / sounds.size());
//        
//        output[i*nChannels    ] = crush;//wave / sounds.size();
//        output[i*nChannels + 1] = crush;//wave / sounds.size();
//    }
//}

//--------------------------------------------------------------
void ofApp::draw(){
//    ofGetElapsedTimeMillis();
    for (int i = 0; i < bands; i++) {
        ofDrawCircle(ofGetWidth() / 2, ofGetHeight() / 2, fft[i] * 100);
    }
}

//--------------------------------------------------------------
void ofApp::keyPressed  (int key){
    switch (key) {
        case 'a':
            sound[0].setVolume(0.99);
            break;
        case 's':
            sound[1].setVolume(0.99);
            break;
        case 'd':
            sound[2].setVolume(0.99);
            break;
        case 'f':
            sound[3].setVolume(0.99);
            break;
        case 'g':
            sound[4].setVolume(0.99);
            break;
        case 'h':
            sound[5].setVolume(0.99);
            break;
        case 'j':
            sound[6].setVolume(0.99);
            break;
        case 'k':
            sound[7].setVolume(0.99);
            break;
        case 'l':
            sound[8].setVolume(0.99);
            break;
    }
}

//--------------------------------------------------------------
void ofApp::keyReleased(int key){
    switch (key) {
        case 'a':
            sound[0].setVolume(0.0);
            break;
        case 's':
            sound[1].setVolume(0.0);
            break;
        case 'd':
            sound[2].setVolume(0.0);
            break;
        case 'f':
            sound[3].setVolume(0.0);
            break;
        case 'g':
            sound[4].setVolume(0.0);
            break;
        case 'h':
            sound[5].setVolume(0.0);
            break;
        case 'j':
            sound[6].setVolume(0.0);
            break;
        case 'k':
            sound[7].setVolume(0.0);
            break;
        case 'l':
            sound[8].setVolume(0.0);
            break;
    }
}

//--------------------------------------------------------------
void ofApp::audioOut( float * output, int bufferSize, int nChannels ) {
    for(int i = 0; i < bufferSize * nChannels; i += 2) {
        float sample = sin(phase); // generating a sine wave sample
        output[i] = sample; // writing to the left channel
        output[i+1] = sample; // writing to the right channel
        phase += 0.05;
    }
}

Any pointers in the right direction will be greatly appreciated. Many thanks.

I’m happy to supply any more information. I’m really stuck on this and am running out of options.
If it’s not possible to compute multiple ffts in real time, I will have to start working in another direction, like recording the output of each sound files fft and “playing” that back alongside the track.
Cheers

Hi,
After a quick read I’m assuming your using ofSoundPlayers and ofSoundGetSpectrum()?
The documentation ofSoundGetSpectrum() says:
Gets a frequency spectrum sample, taking all current sound players into account.
So that’s where you’re at: a mix of all channels…
And as far as I know ofSoundPlayer uses the audio playback of the system, so therefore you can’t access the audio buffers (and perform a FFT).
So I think you’ll have to use a different approach, maybe ofSoundStream? (See examples like soundBufferExample that use a buffer for audio playback) But the FFT becomes a bit more complicated…
This looks promising: https://github.com/roymacdonald/ofxSoundObjects/tree/master/example-ofxFft
Hope this helps.

1 Like