Low-Budget Spectral Envelopes

Some time back I’d written some Java code that attempted to create a rough spectral envelope by picking the n highest point in an FFT. It was moderately successful, but I eventually decided to take the project down the synthesis route.

There have been some requests to see the code. Since WordPress won’t let me attach the source code, I’ve posted the code below. I’ve made a half-hearted attempt to format it, but since WordPress loves to screw up my formatting… Ah, well…

Enjoy! I make no claims that this works, and you’ll obviously need an FFT class to make it usable.

Essentially, the code copies the magnitudes to an array, and then finds the highest point. It then zeros out points around the picked point so no points near that peak will be selected, and then picks another point. It can then linearly interpolate the spectral envelope from the peaks.

It’s not elegant, and there are other well documented algorithms.


import java.awt.Point;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import org.xml.sax.HandlerBase;

public class SpectralEnvelope {

  // sorted list of peaks
  int[] peakBin;

  // values corresponding to the peaks
  double[] peakValue;

  // holds all the envelope values
  double[] envelope;

  /**
   * Create a spectral envelope for the buffer. The buffer
   * is assumed to be FFT data, and only the first half of the
   * data is used. The zeroWidth determines the precision of
   * the envelope. Exact precision (1) creates artifacts.
   * @param buffer FFT buffer containing data.
   * @param wide Number of sample points to use
   * @param peakCount Number of peaks to gather
   */
   public void build(double[] buffer, int wide, int peakCount) {
     // clear the envelope
    envelope = new double[wide];

    // copy of the data, because used peaks will have to be flagged
    double[] tmp = new double[wide];
    System.arraycopy(buffer, 0, tmp, 0, wide);

    // list of peak indexes
    peakBin = new int[peakCount];

    // add the first and last points
    peakBin[0] = 0;
    peakBin[1] = wide - 1;

    int peaksFound = 2;

    // estimate width needed to get requested peak count
    int zeroWidth = (int)(wide / peakCount / 2);

    // identify peaks, starting from the highest
    while (peaksFound < peakCount) {
      // clear the max
      double max = 0;
      int maxAt = 0;

      // search for the highest value
      for (int j = 0; j < wide; j++) { if (tmp[j] > max) {
        max = tmp[j];
        maxAt = j;
      }
    }

    // all peaks are found, exit
    if (max == 0) break;

    // store peak and increment peak count
    peakBin[peaksFound-1] = maxAt;
    peaksFound++;

    // zero area around peak so they will be ignored
    int zeroStart = Math.max(maxAt-zeroWidth, 0);
    int zeroEnd = Math.min(maxAt+zeroWidth+1, tmp.length);
    for (int index = zeroStart; index < zeroEnd ; index++) {
      tmp[index] = 0;
    }
  }

  // sort the peaks
  Arrays.sort(peakBin);

  // store the values of the peaks
  peakValue = new double[peakCount];
  for (int i = 0; i < peakCount; i++) {
    // get the value from the array and store it
    peakValue[i] = buffer[peakBin[i]];
  }

  // connect the peaks
  double delta = 0;
  for (int i = 1; i < peakBin.length; i++) {
    // get the next peak
    int thisBinIndex = peakBin[i];
    int priorBinIndex = peakBin[i-1];

    // determine the slope
    int dy = thisBinIndex - priorBinIndex;

    // handle division by zero
    if (dy == 0) {
      delta = 0;
    } else {
      delta = (peakValue[i] - peakValue[i-1]) / dy;
    }

    // set the magnitudes
    double m = peakValue[i-1];
    for (int j = priorBinIndex; j < thisBinIndex; j++) {
      envelope[j] = m;
      m += delta;
    }
  }

  }

  /**
  * Linearly interpolates a spectral envelope between the source and
  * target envelopes.
  * @param source Initial envelope (amount = 0)
  * @param target Target envelope (amount = 1)
  * @param amount Range to interpolate value at (0..1)
  */
  void interpolateEnvelope(SpectralEnvelope source, SpectralEnvelope target, double amount) {

  // number of peaks
  int peakCount = target.peakBin.length;

  // create an array to hold the interpolated peaks
  int[] lerpBin = new int[peakCount];

  // create a hashtable to hold the interpolated peak values
  Hashtable<Integer,Double> lerpValue = new Hashtable<Integer,Double>();

  // interpolate the values
  for (int i = 1; i < peakCount; i++) {
    // interpolated bin number
    lerpBin[i] = (int)Util.lerp(source.peakBin[i], target.peakBin[i], amount);
    // interpolated value
    lerpValue.put(lerpBin[i], Util.lerp(source.peakValue[i], target.peakValue[i], amount));
   }

  // clear the envelope
  envelope = new double[source.envelope.length];

  // sort the bins
  Arrays.sort(lerpBin);

  // create the spectral envelopes using the interpolated values
  double priorValue = lerpValue.get(lerpBin[0]);
  double delta = 0;
  for (int i = 1; i < peakCount; i++) {
    // get the next peak
   int thisBinIndex = lerpBin[i];
   int priorBinIndex = lerpBin[i-1];

   // get this value
  double thisValue = lerpValue.get(lerpBin[i]);

  // determine the slope
  int dy = thisBinIndex - priorBinIndex;

  // handle divsion by zero
  if (dy == 0) {
    delta = 0;
  } else {
    // calculate slope
    delta = (thisValue - priorValue) / dy;
  }

  // set the magnitudes
  double m = priorValue;
  for (int j = priorBinIndex; j < thisBinIndex; j++) { 
    envelope[j] = m; 
    m += delta; 
  } 

  // save current as prior value 
  priorValue = thisValue; 

  } 
} 

/** * Since a single sample isn't likely to create an accurate spectral 
* envelope, this function is used to accumulate the maximum values in 
* over a number of samples. 
* @param fft The fft 
* @param buffer Buffer with the data to sample 
* @param pos Starting position 
* @param steps Number of samples 
* @param stepSize Distance between the samples 
* @return 
* @throws Exception 
*/ 
final static double[] accumulateSteps(ProcessFFT fft, double[] buffer, int pos, int steps, int stepSize ) throws Exception { 
  // buffer holding the results 
  double[] accum = new double[fft.frameSize2]; 
  // iterate through the buffer, decrementing the count 
  for (; steps > 0; steps--, pos += stepSize ) {
    // process the fft
    fft.analyze(buffer, pos);

    // save the high values to the accumulator
    for (int i = 0; i < fft.frameSize2; i++) {
      accum[i] = Math.max(accum[i], fft.gAnaMagn[i]);
    }
  }
  return accum;
 }
}

The call to the pitch shifting looks something like this:

 

    /**
     * Pitch shifting with formant preservation
     */
    public void pitchShiftWithFormantPreservation() {
        
        // create a spectral envelope
        this.spectralEnvelope = new SpectralEnvelope();
        
        // build the spectral envelope
        spectralEnvelope.build(this.gAnaMagn, this.frameSize2, 100);
        
        // get the envelope
        double[] envelope = spectralEnvelope.envelope;
        
        // transpose the pitch
        for (int i = 0; i < this.frameSize2; i++) {
            // get the index of the target
            int target = (int)(i * this.pitchShift);
            
            // exit loop if past the end
            if (target >= this.frameSize2) break;
            
            // scale relative to the envelope            
            // need to check for division by zeros
            if (envelope[i] == 0 || envelope[target] == 0) {
                gSynMagn[target] = 0;
            } else {
                gSynMagn[target] = this.gAnaMagn[i] / envelope[i] * envelope[target];
            }
            gSynFreq[target] = gAnaFreq[i] * this.pitchShift;    
            
        }
    }


 

Advertisements

About synsinger

Developer and Musician
This entry was posted in Development and tagged . Bookmark the permalink.

One Response to Low-Budget Spectral Envelopes

  1. Denis Louis says:

    You can’t imagine how gratefull I am for this ! 🙂

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s