Some time back I’d written some Java code that attempted to create a rough spectral envelope by picking the n highest point in an FFT. It was moderately successful, but I eventually decided to take the project down the synthesis route.
There have been some requests to see the code. Since WordPress won’t let me attach the source code, I’ve posted the code below. I’ve made a half-hearted attempt to format it, but since WordPress loves to screw up my formatting… Ah, well…
Enjoy! I make no claims that this works, and you’ll obviously need an FFT class to make it usable.
Essentially, the code copies the magnitudes to an array, and then finds the highest point. It then zeros out points around the picked point so no points near that peak will be selected, and then picks another point. It can then linearly interpolate the spectral envelope from the peaks.
It’s not elegant, and there are other well documented algorithms.
import java.awt.Point;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import org.xml.sax.HandlerBase;
public class SpectralEnvelope {
// sorted list of peaks
int[] peakBin;
// values corresponding to the peaks
double[] peakValue;
// holds all the envelope values
double[] envelope;
/**
* Create a spectral envelope for the buffer. The buffer
* is assumed to be FFT data, and only the first half of the
* data is used. The zeroWidth determines the precision of
* the envelope. Exact precision (1) creates artifacts.
* @param buffer FFT buffer containing data.
* @param wide Number of sample points to use
* @param peakCount Number of peaks to gather
*/
public void build(double[] buffer, int wide, int peakCount) {
// clear the envelope
envelope = new double[wide];
// copy of the data, because used peaks will have to be flagged
double[] tmp = new double[wide];
System.arraycopy(buffer, 0, tmp, 0, wide);
// list of peak indexes
peakBin = new int[peakCount];
// add the first and last points
peakBin[0] = 0;
peakBin[1] = wide - 1;
int peaksFound = 2;
// estimate width needed to get requested peak count
int zeroWidth = (int)(wide / peakCount / 2);
// identify peaks, starting from the highest
while (peaksFound < peakCount) {
// clear the max
double max = 0;
int maxAt = 0;
// search for the highest value
for (int j = 0; j < wide; j++) { if (tmp[j] > max) {
max = tmp[j];
maxAt = j;
}
}
// all peaks are found, exit
if (max == 0) break;
// store peak and increment peak count
peakBin[peaksFound-1] = maxAt;
peaksFound++;
// zero area around peak so they will be ignored
int zeroStart = Math.max(maxAt-zeroWidth, 0);
int zeroEnd = Math.min(maxAt+zeroWidth+1, tmp.length);
for (int index = zeroStart; index < zeroEnd ; index++) {
tmp[index] = 0;
}
}
// sort the peaks
Arrays.sort(peakBin);
// store the values of the peaks
peakValue = new double[peakCount];
for (int i = 0; i < peakCount; i++) {
// get the value from the array and store it
peakValue[i] = buffer[peakBin[i]];
}
// connect the peaks
double delta = 0;
for (int i = 1; i < peakBin.length; i++) {
// get the next peak
int thisBinIndex = peakBin[i];
int priorBinIndex = peakBin[i-1];
// determine the slope
int dy = thisBinIndex - priorBinIndex;
// handle division by zero
if (dy == 0) {
delta = 0;
} else {
delta = (peakValue[i] - peakValue[i-1]) / dy;
}
// set the magnitudes
double m = peakValue[i-1];
for (int j = priorBinIndex; j < thisBinIndex; j++) {
envelope[j] = m;
m += delta;
}
}
}
/**
* Linearly interpolates a spectral envelope between the source and
* target envelopes.
* @param source Initial envelope (amount = 0)
* @param target Target envelope (amount = 1)
* @param amount Range to interpolate value at (0..1)
*/
void interpolateEnvelope(SpectralEnvelope source, SpectralEnvelope target, double amount) {
// number of peaks
int peakCount = target.peakBin.length;
// create an array to hold the interpolated peaks
int[] lerpBin = new int[peakCount];
// create a hashtable to hold the interpolated peak values
Hashtable<Integer,Double> lerpValue = new Hashtable<Integer,Double>();
// interpolate the values
for (int i = 1; i < peakCount; i++) {
// interpolated bin number
lerpBin[i] = (int)Util.lerp(source.peakBin[i], target.peakBin[i], amount);
// interpolated value
lerpValue.put(lerpBin[i], Util.lerp(source.peakValue[i], target.peakValue[i], amount));
}
// clear the envelope
envelope = new double[source.envelope.length];
// sort the bins
Arrays.sort(lerpBin);
// create the spectral envelopes using the interpolated values
double priorValue = lerpValue.get(lerpBin[0]);
double delta = 0;
for (int i = 1; i < peakCount; i++) {
// get the next peak
int thisBinIndex = lerpBin[i];
int priorBinIndex = lerpBin[i-1];
// get this value
double thisValue = lerpValue.get(lerpBin[i]);
// determine the slope
int dy = thisBinIndex - priorBinIndex;
// handle divsion by zero
if (dy == 0) {
delta = 0;
} else {
// calculate slope
delta = (thisValue - priorValue) / dy;
}
// set the magnitudes
double m = priorValue;
for (int j = priorBinIndex; j < thisBinIndex; j++) {
envelope[j] = m;
m += delta;
}
// save current as prior value
priorValue = thisValue;
}
}
/** * Since a single sample isn't likely to create an accurate spectral
* envelope, this function is used to accumulate the maximum values in
* over a number of samples.
* @param fft The fft
* @param buffer Buffer with the data to sample
* @param pos Starting position
* @param steps Number of samples
* @param stepSize Distance between the samples
* @return
* @throws Exception
*/
final static double[] accumulateSteps(ProcessFFT fft, double[] buffer, int pos, int steps, int stepSize ) throws Exception {
// buffer holding the results
double[] accum = new double[fft.frameSize2];
// iterate through the buffer, decrementing the count
for (; steps > 0; steps--, pos += stepSize ) {
// process the fft
fft.analyze(buffer, pos);
// save the high values to the accumulator
for (int i = 0; i < fft.frameSize2; i++) {
accum[i] = Math.max(accum[i], fft.gAnaMagn[i]);
}
}
return accum;
}
}
The call to the pitch shifting looks something like this:
/**
* Pitch shifting with formant preservation
*/
public void pitchShiftWithFormantPreservation() {
// create a spectral envelope
this.spectralEnvelope = new SpectralEnvelope();
// build the spectral envelope
spectralEnvelope.build(this.gAnaMagn, this.frameSize2, 100);
// get the envelope
double[] envelope = spectralEnvelope.envelope;
// transpose the pitch
for (int i = 0; i < this.frameSize2; i++) {
// get the index of the target
int target = (int)(i * this.pitchShift);
// exit loop if past the end
if (target >= this.frameSize2) break;
// scale relative to the envelope
// need to check for division by zeros
if (envelope[i] == 0 || envelope[target] == 0) {
gSynMagn[target] = 0;
} else {
gSynMagn[target] = this.gAnaMagn[i] / envelope[i] * envelope[target];
}
gSynFreq[target] = gAnaFreq[i] * this.pitchShift;
}
}
