I am trying to set up both an FFTTap and a PitchTap to an audioEngine along with an oscillator in a Swift SpriteKit game. The aim is to process both pitch and amplitude of mic input via PitchTap and harmonics and other data from the mic via FFTTap while providing headphone audio to the user via the oscillator. Audio processing needs to be quite accurate and in real time. I have successfully processed PitchTap data with my current setup so the fundamentals of the audio configuration are sound. However attempting to add the FFTTap to the graph has proved difficult.
Here is the relevant current code:
class AudioEngineManager {
static let shared = AudioEngineManager()
var audioEngine = AudioEngine()
var mixer = Mixer() // Main mixer for output
var oscillator = Oscillator() // Oscillator for generating tones
var inputMixer = Mixer() // Mixer for managing input sources
private init() {
setupAudioComponents()
}
func configureAudioSession() {
do {
let audioSession = AVAudioSession.sharedInstance()
try audioSession.setPreferredSampleRate(44100)
try audioSession.setPreferredIOBufferDuration(0.005)
try audioSession.setCategory(.playAndRecord, mode: .measurement, options: [.defaultToSpeaker, .allowBluetoothA2DP])
try audioSession.setActive(true)
print("Audio session configured.")
} catch {
print("Failed to configure audio session: (error)")
}
}
private func setupAudioComponents() {
oscillator.amplitude = 0
oscillator.start()
if let input = audioEngine.input {
inputMixer.addInput(input)
// Ensure the mixer handles stereo format if needed
}
mixer.addInput(oscillator)
mixer.addInput(inputMixer) // Ensure inputMixer is added to the main mixer
// Optionally, handle output format explicitly
audioEngine.output = mixer
}
func startAudioEngine() {
do {
inputMixer.start() // Ensure inputMixer is also started
mixer.start() // Starting the main mixer
try audioEngine.start()
let mixerFormat = mixer.avAudioNode.outputFormat(forBus: 0)
print("Number of channels: (mixerFormat.channelCount)")
print("Audio engine started successfully.")
} catch {
print("Failed to start audio engine: (error)")
}
}
func stopAudioEngine() {
audioEngine.stop()
print("Audio engine stopped.")
}
func playTone(frequency: Double, amplitude: Double, duration: TimeInterval) {
oscillator.frequency = AUValue(frequency)
oscillator.amplitude = AUValue(amplitude)
DispatchQueue.main.asyncAfter(deadline: .now() + duration) {
self.stopTone()
}
}
func stopTone() {
oscillator.amplitude = 0
}
}
class PitchTracker {
static let shared = PitchTracker()
private var audioEngine: AudioEngine
private var inputMixer: Mixer
private var fftInput: Mixer
private var pitchInput: Mixer
var pitchTap: PitchTap?
var fftTap: FFTTap?
var audioDetectedHandler: ((AudioData) -> Void)?
init() {
self.audioEngine = AudioEngineManager.shared.audioEngine
self.inputMixer = AudioEngineManager.shared.inputMixer
self.fftInput = Mixer()
self.pitchInput = Mixer()
setupTaps()
connectAudioPaths()
}
private func setupTaps() {
fftTap = FFTTap(fftInput, bufferSize: 4096) { fftData in
// Handle FFT data
print("FFT Data received.")
}
pitchTap = PitchTap(pitchInput) { pitch, amp in
// Handle pitch data
print("Pitch: (pitch[0]), Amplitude: (amp[0])")
}
}
private func connectAudioPaths() {
inputMixer.addInput(fftInput)
inputMixer.addInput(pitchInput)
// Ensure the inputs are part of the audio graph
audioEngine.output = inputMixer
}
func startTracking() {
do {
fftTap?.start()
pitchTap?.start()
inputMixer.start()
print("Audio tracking started.")
} catch {
print("Failed to start taps: (error)")
}
}
func stopTracking() {
fftTap?.stop()
pitchTap?.stop()
print("Audio tracking stopped.")
}
func differenceInCents(frequency1: Float, frequency2: Float) -> Float {
return 1200 * log2(frequency1 / frequency2)
}
}
struct AudioData {
var pitch: CGFloat
var amplitude: CGFloat
var harmonicsRatios: [CGFloat] // Fundamental, First Harmonic, and Second Harmonic
}
I configure the AudioSession and start the singleton AudioEngine in appDelegate.
This code produces errors: ioData.mNumberBuffers=2, ASBD::NumberChannelStreams(output.GetStreamFormat())=1; kAudio_ParamError
from AU (0x102a12600): auou/rioc/appl, render err: -50
I would welcome any fix to the above code or suggestions for an alternative approach.