geemili
/
SDL
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

audio: Added a postmix callback to logical devices. 

You can see it in action in testaudio by mousing over a logical device; it
will show a visualizer for the current PCM (whatever is currently being
recorded on a capture device, or whatever is being mixed for output on
playback devices).

Fixes #8122.
main
Ryan C. Gordon 2023-09-07 10:56:09 -04:00
parent 7207bdce5d
commit 3a992af446
7 changed files with 316 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
include/SDL3/SDL_audio.h
View File

@ -1126,6 +1126,73 @@ extern DECLSPEC void SDLCALL SDL_DestroyAudioStream(SDL_AudioStream *stream);
*/
extern DECLSPEC SDL_AudioStream *SDLCALL SDL_OpenAudioDeviceStream(SDL_AudioDeviceID devid, const SDL_AudioSpec *spec, SDL_AudioStreamCallback callback, void *userdata);
/**
* A callback that fires when data is about to be fed to an audio device.
*
* This is useful for accessing the final mix, perhaps for writing a
* visualizer or applying a final effect to the audio data before playback.
*
* \sa SDL_SetAudioDevicePostmixCallback
*/
typedef void (SDLCALL *SDL_AudioPostmixCallback)(void *userdata, const SDL_AudioSpec *spec, float *buffer, int buflen);
/**
* Set a callback that fires when data is about to be fed to an audio device.
*
* This is useful for accessing the final mix, perhaps for writing a
* visualizer or applying a final effect to the audio data before playback.
*
* The buffer is the final mix of all bound audio streams on an opened
* device; this callback will fire regularly for any device that is both
* opened and unpaused. If there is no new data to mix, either because no
* streams are bound to the device or all the streams are empty, this
* callback will still fire with the entire buffer set to silence.
*
* This callback is allowed to make changes to the data; the contents of
* the buffer after this call is what is ultimately passed along to the
* hardware.
*
* The callback is always provided the data in float format (values from
* -1.0f to 1.0f), but the number of channels or sample rate may be
* different than the format the app requested when opening the device; SDL
* might have had to manage a conversion behind the scenes, or the playback
* might have jumped to new physical hardware when a system default changed,
* etc. These details may change between calls. Accordingly, the size of the
* buffer might change between calls as well.
*
* This callback can run at any time, and from any thread; if you need to
* serialize access to your app's data, you should provide and use a mutex or
* other synchronization device.
*
* All of this to say: there are specific needs this callback can fulfill,
* but it is not the simplest interface. Apps should generally provide audio
* in their preferred format through an SDL_AudioStream and let SDL handle
* the difference.
*
* This function is extremely time-sensitive; the callback should do the
* least amount of work possible and return as quickly as it can. The longer
* the callback runs, the higher the risk of audio dropouts or other problems.
*
* This function will block until the audio device is in between iterations,
* so any existing callback that might be running will finish before this
* function sets the new callback and returns.
*
* Setting a NULL callback function disables any previously-set callback.
*
* \param devid The ID of an opened audio device.
* \param callback A callback function to be called. Can be NULL.
* \param userdata App-controlled pointer passed to callback. Can be NULL.
* \returns zero on success, -1 on error; call SDL_GetError() for more
* information.
*
* \threadsafety It is safe to call this function from any thread.
*
* \since This function is available since SDL 3.0.0.
*/
extern DECLSPEC int SDLCALL SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata);
/**
* Load the audio data of a WAVE file into memory.
*

81
src/audio/SDL_audio.c
View File

@ -715,6 +715,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
// can we do a basic copy without silencing/mixing the buffer? This is an extremely likely scenario, so we special-case it.
const SDL_bool simple_copy = device->logical_devices && // there's a logical device
!device->logical_devices->next && // there's only _ONE_ logical device
!device->logical_devices->postmix && // there isn't a postmix callback
!SDL_AtomicGet(&device->logical_devices->paused) && // it isn't paused
device->logical_devices->bound_streams && // there's a bound stream
!device->logical_devices->bound_streams->next_binding; // there's only _ONE_ bound stream.
@ -731,7 +732,7 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
SDL_memset(device_buffer + br, device->silence_value, buffer_size - br); // silence whatever we didn't write to.
}
} else { // need to actually mix (or silence the buffer)
float *mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer);
float *final_mix_buffer = (float *) ((device->spec.format == SDL_AUDIO_F32) ? device_buffer : device->mix_buffer);
const int needed_samples = buffer_size / SDL_AUDIO_BYTESIZE(device->spec.format);
const int work_buffer_size = needed_samples * sizeof (float);
SDL_AudioSpec outspec;
@ -742,13 +743,20 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
outspec.channels = device->spec.channels;
outspec.freq = device->spec.freq;
SDL_memset(mix_buffer, '\0', buffer_size); // start with silence.
SDL_memset(final_mix_buffer, '\0', work_buffer_size); // start with silence.
for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) {
if (SDL_AtomicGet(&logdev->paused)) {
continue; // paused? Skip this logical device.
}
const SDL_AudioPostmixCallback postmix = logdev->postmix;
float *mix_buffer = final_mix_buffer;
if (postmix) {
mix_buffer = device->postmix_buffer;
SDL_memset(mix_buffer, '\0', work_buffer_size); // start with silence.
}
for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) {
SDL_SetAudioStreamFormat(stream, NULL, &outspec);
@ -764,12 +772,18 @@ SDL_bool SDL_OutputAudioThreadIterate(SDL_AudioDevice *device)
MixFloat32Audio(mix_buffer, (float *) device->work_buffer, br);
}
}
if (postmix) {
SDL_assert(mix_buffer == device->postmix_buffer);
postmix(logdev->postmix_userdata, &outspec, mix_buffer, work_buffer_size);
MixFloat32Audio(final_mix_buffer, mix_buffer, work_buffer_size);
}
}
if (((Uint8 *) mix_buffer) != device_buffer) {
if (((Uint8 *) final_mix_buffer) != device_buffer) {
// !!! FIXME: we can't promise the device buf is aligned/padded for SIMD.
//ConvertAudio(needed_samples * device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer);
ConvertAudio(needed_samples / device->spec.channels, mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL);
//ConvertAudio(needed_samples * device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device_buffer, device->spec.format, device->spec.channels, device->work_buffer);
ConvertAudio(needed_samples / device->spec.channels, final_mix_buffer, SDL_AUDIO_F32, device->spec.channels, device->work_buffer, device->spec.format, device->spec.channels, NULL);
SDL_memcpy(device_buffer, device->work_buffer, buffer_size);
}
}
@ -837,21 +851,37 @@ SDL_bool SDL_CaptureAudioThreadIterate(SDL_AudioDevice *device)
current_audio.impl.FlushCapture(device); // nothing wants data, dump anything pending.
} else {
// this SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitCaptureDevice!
const int rc = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size);
if (rc < 0) { // uhoh, device failed for some reason!
int br = current_audio.impl.CaptureFromDevice(device, device->work_buffer, device->buffer_size);
if (br < 0) { // uhoh, device failed for some reason!
retval = SDL_FALSE;
} else if (rc > 0) { // queue the new data to each bound stream.
} else if (br > 0) { // queue the new data to each bound stream.
for (SDL_LogicalAudioDevice *logdev = device->logical_devices; logdev != NULL; logdev = logdev->next) {
if (SDL_AtomicGet(&logdev->paused)) {
continue; // paused? Skip this logical device.
}
void *output_buffer = device->work_buffer;
SDL_AudioSpec outspec;
SDL_copyp(&outspec, &device->spec);
// I don't know why someone would want a postmix on a capture device, but we offer it for API consistency.
if (logdev->postmix) {
// move to float format.
output_buffer = device->postmix_buffer;
outspec.format = SDL_AUDIO_F32;
const int frames = br / SDL_AUDIO_FRAMESIZE(device->spec);
br = frames * SDL_AUDIO_FRAMESIZE(outspec);
ConvertAudio(frames, device->work_buffer, device->spec.format, outspec.channels, device->postmix_buffer, SDL_AUDIO_F32, outspec.channels, NULL);
logdev->postmix(logdev->postmix_userdata, &outspec, device->postmix_buffer, br);
}
for (SDL_AudioStream *stream = logdev->bound_streams; stream != NULL; stream = stream->next_binding) {
/* this will hold a lock on `stream` while putting. We don't explicitly lock the streams
for iterating here because the binding linked list can only change while the device lock is held.
(we _do_ lock the stream during binding/unbinding to make sure that two threads can't try to bind
the same stream to different devices at the same time, though.) */
if (SDL_PutAudioStreamData(stream, device->work_buffer, rc) < 0) {
SDL_SetAudioStreamFormat(stream, &outspec, NULL);
if (SDL_PutAudioStreamData(stream, output_buffer, br) < 0) {
// oh crud, we probably ran out of memory. This is possibly an overreaction to kill the audio device, but it's likely the whole thing is going down in a moment anyhow.
retval = SDL_FALSE;
break;
@ -1138,6 +1168,9 @@ static void ClosePhysicalAudioDevice(SDL_AudioDevice *device)
SDL_aligned_free(device->mix_buffer);
device->mix_buffer = NULL;
SDL_aligned_free(device->postmix_buffer);
device->postmix_buffer = NULL;
SDL_memcpy(&device->spec, &device->default_spec, sizeof (SDL_AudioSpec));
device->sample_frames = 0;
device->silence_value = SDL_GetSilenceValueForFormat(device->spec.format);
@ -1395,6 +1428,28 @@ SDL_bool SDL_IsAudioDevicePaused(SDL_AudioDeviceID devid)
return retval;
}
int SDL_SetAudioPostmixCallback(SDL_AudioDeviceID devid, SDL_AudioPostmixCallback callback, void *userdata)
{
SDL_LogicalAudioDevice *logdev = ObtainLogicalAudioDevice(devid);
int retval = 0;
if (logdev) {
SDL_AudioDevice *device = logdev->physical_device;
if (!device->postmix_buffer) {
device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size);
if (device->mix_buffer == NULL) {
retval = SDL_OutOfMemory();
}
}
if (retval == 0) {
logdev->postmix = callback;
logdev->postmix_userdata = userdata;
}
SDL_UnlockMutex(device->lock);
}
return retval;
}
int SDL_BindAudioStreams(SDL_AudioDeviceID devid, SDL_AudioStream **streams, int num_streams)
{
@ -1782,6 +1837,14 @@ int SDL_AudioDeviceFormatChangedAlreadyLocked(SDL_AudioDevice *device, const SDL
kill_device = SDL_TRUE;
}
if (device->postmix_buffer) {
SDL_aligned_free(device->postmix_buffer);
device->postmix_buffer = (float *)SDL_aligned_alloc(SDL_SIMDGetAlignment(), device->work_buffer_size);
if (!device->postmix_buffer) {
kill_device = SDL_TRUE;
}
}
SDL_aligned_free(device->mix_buffer);
device->mix_buffer = NULL;
if (device->spec.format != SDL_AUDIO_F32) {

7
src/audio/SDL_sysaudio.h
View File

@ -214,6 +214,12 @@ struct SDL_LogicalAudioDevice
// SDL_TRUE if device was opened with SDL_OpenAudioDeviceStream (so it forbids binding changes, etc).
SDL_bool simplified;
// If non-NULL, callback into the app that lets them access the final postmix buffer.
SDL_AudioPostmixCallback postmix;
// App-supplied pointer for postmix callback.
void *postmix_userdata;
// double-linked list of opened devices on the same physical device.
SDL_LogicalAudioDevice *next;
SDL_LogicalAudioDevice *prev;
@ -264,6 +270,7 @@ struct SDL_AudioDevice
// Scratch buffers used for mixing.
Uint8 *work_buffer;
Uint8 *mix_buffer;
float *postmix_buffer;
// Size of work_buffer (and mix_buffer) in bytes.
int work_buffer_size;

1
src/dynapi/SDL_dynapi.sym
View File

@ -904,6 +904,7 @@ SDL3_0.0.0 {
SDL_SetWindowFocusable;
SDL_GetAudioStreamFrequencyRatio;
SDL_SetAudioStreamFrequencyRatio;
SDL_SetAudioPostmixCallback;
# extra symbols go here (don't modify this line)
local: *;
};

1
src/dynapi/SDL_dynapi_overrides.h
View File

@ -929,3 +929,4 @@
#define SDL_SetWindowFocusable SDL_SetWindowFocusable_REAL
#define SDL_GetAudioStreamFrequencyRatio SDL_GetAudioStreamFrequencyRatio_REAL
#define SDL_SetAudioStreamFrequencyRatio SDL_SetAudioStreamFrequencyRatio_REAL
#define SDL_SetAudioPostmixCallback SDL_SetAudioPostmixCallback_REAL

1
src/dynapi/SDL_dynapi_procs.h
View File

@ -975,3 +975,4 @@ SDL_DYNAPI_PROC(int,SDL_GDKGetDefaultUser,(XUserHandle *a),(a),return)
SDL_DYNAPI_PROC(int,SDL_SetWindowFocusable,(SDL_Window *a, SDL_bool b),(a,b),return)
SDL_DYNAPI_PROC(float,SDL_GetAudioStreamFrequencyRatio,(SDL_AudioStream *a),(a),return)
SDL_DYNAPI_PROC(int,SDL_SetAudioStreamFrequencyRatio,(SDL_AudioStream *a, float b),(a,b),return)
SDL_DYNAPI_PROC(int,SDL_SetAudioPostmixCallback,(SDL_AudioDeviceID a, SDL_AudioPostmixCallback b, void *c),(a,b,c),return)

167
test/testaudio.c
View File

@ -10,6 +10,8 @@
#include "testutils.h"
#define POOF_LIFETIME 250
#define VISUALIZER_WIDTH 100
#define VISUALIZER_HEIGHT 50
typedef struct Texture
{
@ -50,6 +52,15 @@ struct Thing
SDL_bool iscapture;
SDL_AudioSpec spec;
Thing *physdev;
SDL_bool visualizer_enabled;
SDL_bool visualizer_updated;
SDL_Texture *visualizer;
SDL_Mutex *postmix_lock;
float *postmix_buffer;
int postmix_buflen;
int postmix_allocated;
SDL_AudioSpec postmix_spec;
SDL_AtomicInt postmix_updated;
} logdev;
struct {
SDL_AudioSpec spec;
@ -98,6 +109,7 @@ static SDLTest_CommonState *state = NULL;
static Thing *things = NULL;
static char *current_titlebar = NULL;
static Thing *mouseover_thing = NULL;
static Thing *droppable_highlighted_thing = NULL;
static Thing *dragging_thing = NULL;
static int dragging_button = -1;
@ -279,21 +291,40 @@ static void DestroyThing(Thing *thing)
return;
}
if (mouseover_thing == thing) {
mouseover_thing = NULL;
}
if (droppable_highlighted_thing == thing) {
droppable_highlighted_thing = NULL;
}
if (dragging_thing == thing) {
dragging_thing = NULL;
}
switch (thing->what) {
case THING_POOF: break;
case THING_NULL: break;
case THING_TRASHCAN: break;
case THING_LOGDEV:
case THING_LOGDEV_CAPTURE:
SDL_CloseAudioDevice(thing->data.logdev.devid);
SDL_DestroyTexture(thing->data.logdev.visualizer);
SDL_DestroyMutex(thing->data.logdev.postmix_lock);
SDL_free(thing->data.logdev.postmix_buffer);
break;
case THING_PHYSDEV:
case THING_PHYSDEV_CAPTURE:
SDL_free(thing->data.physdev.name);
break;
case THING_WAV:
SDL_free(thing->data.wav.buf);
break;
case THING_STREAM:
SDL_DestroyAudioStream(thing->data.stream.stream);
break;
@ -748,6 +779,135 @@ static void LogicalDeviceThing_ondrop(Thing *thing, int button, float x, float y
}
}
static void SDLCALL PostmixCallback(void *userdata, const SDL_AudioSpec *spec, float *buffer, int buflen)
{
Thing *thing = (Thing *) userdata;
SDL_LockMutex(thing->data.logdev.postmix_lock);
if (thing->data.logdev.postmix_allocated < buflen) {
void *ptr = SDL_realloc(thing->data.logdev.postmix_buffer, buflen);
if (!ptr) {
SDL_UnlockMutex(thing->data.logdev.postmix_lock);
return; /* oh well. */
}
thing->data.logdev.postmix_buffer = (float *) ptr;
thing->data.logdev.postmix_allocated = buflen;
}
SDL_copyp(&thing->data.logdev.postmix_spec, spec);
SDL_memcpy(thing->data.logdev.postmix_buffer, buffer, buflen);
thing->data.logdev.postmix_buflen = buflen;
SDL_AtomicSet(&thing->data.logdev.postmix_updated, 1);
SDL_UnlockMutex(thing->data.logdev.postmix_lock);
}
static void UpdateVisualizer(SDL_Renderer *renderer, SDL_Texture *visualizer, const int channels, const float *buffer, const int buflen)
{
static const SDL_Color channel_colors[8] = {
{ 255, 255, 255, 255 },
{ 255, 0, 0, 255 },
{ 0, 255, 0, 255 },
{ 0, 0, 255, 255 },
{ 255, 255, 0, 255 },
{ 0, 255, 255, 255 },
{ 255, 0, 255, 255 },
{ 127, 127, 127, 255 }
};
SDL_SetRenderTarget(renderer, visualizer);
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 0);
SDL_RenderClear(renderer);
if (buffer && buflen) {
const int frames = (buflen / sizeof (float)) / channels;
const int skip = frames / (VISUALIZER_WIDTH * 2);
int i, j;
for (i = channels - 1; i >= 0; i--) {
const SDL_Color *color = &channel_colors[i % SDL_arraysize(channel_colors)];
SDL_FPoint points[VISUALIZER_WIDTH + 2];
float prevx = 0.0f;
int pointidx = 1;
points[0].x = 0.0f;
points[0].y = VISUALIZER_HEIGHT * 0.5f;
for (j = 0; j < (SDL_arraysize(points)-1); j++) {
const float val = buffer[((j * skip) * channels) + i];
const float x = prevx + 2;
const float y = (VISUALIZER_HEIGHT * 0.5f) - (VISUALIZER_HEIGHT * (val * 0.5f));
SDL_assert(pointidx < SDL_arraysize(points));
points[pointidx].x = x;
points[pointidx].y = y;
pointidx++;
prevx = x;
}
SDL_SetRenderDrawColor(renderer, color->r, color->g, color->b, 255);
SDL_RenderLines(renderer, points, pointidx);
}
}
SDL_SetRenderTarget(renderer, NULL);
}
static void LogicalDeviceThing_ontick(Thing *thing, Uint64 now)
{
const SDL_bool ismousedover = (thing == mouseover_thing) ? SDL_TRUE : SDL_FALSE;
if (!thing->data.logdev.visualizer || !thing->data.logdev.postmix_lock) { /* need these to work, skip if they failed. */
return;
}
if (thing->data.logdev.visualizer_enabled != ismousedover) {
thing->data.logdev.visualizer_enabled = ismousedover;
if (!ismousedover) {
SDL_SetAudioPostmixCallback(thing->data.logdev.devid, NULL, NULL);
} else {
if (thing->data.logdev.postmix_buffer) {
SDL_memset(thing->data.logdev.postmix_buffer, '\0', thing->data.logdev.postmix_buflen);
}
SDL_AtomicSet(&thing->data.logdev.postmix_updated, 1); /* so this will at least clear the texture later. */
SDL_SetAudioPostmixCallback(thing->data.logdev.devid, PostmixCallback, thing);
}
}
}
static void LogicalDeviceThing_ondraw(Thing *thing, SDL_Renderer *renderer)
{
if (thing->data.logdev.visualizer_enabled) {
SDL_FRect dst;
dst.w = thing->rect.w;
dst.h = thing->rect.h;
dst.x = thing->rect.x + ((thing->rect.w - dst.w) / 2);
dst.y = thing->rect.y + ((thing->rect.h - dst.h) / 2);
if (SDL_AtomicGet(&thing->data.logdev.postmix_updated)) {
float *buffer;
int channels;
int buflen;
SDL_LockMutex(thing->data.logdev.postmix_lock);
channels = thing->data.logdev.postmix_spec.channels;
buflen = thing->data.logdev.postmix_buflen;
buffer = (float *) SDL_malloc(thing->data.logdev.postmix_buflen);
if (buffer) {
SDL_memcpy(buffer, thing->data.logdev.postmix_buffer, thing->data.logdev.postmix_buflen);
SDL_AtomicSet(&thing->data.logdev.postmix_updated, 0);
}
SDL_UnlockMutex(thing->data.logdev.postmix_lock);
UpdateVisualizer(renderer, thing->data.logdev.visualizer, channels, buffer, buflen);
SDL_free(buffer);
}
SDL_SetRenderDrawColor(renderer, 255, 255, 255, 30);
SDL_RenderTexture(renderer, thing->data.logdev.visualizer, NULL, &dst);
}
}
static Thing *CreateLogicalDeviceThing(Thing *parent, const SDL_AudioDeviceID which, const float x, const float y)
{
static const ThingType can_be_dropped_onto[] = { THING_TRASHCAN, THING_NULL };
@ -760,9 +920,16 @@ static Thing *CreateLogicalDeviceThing(Thing *parent, const SDL_AudioDeviceID wh
thing->data.logdev.devid = which;
thing->data.logdev.iscapture = iscapture;
thing->data.logdev.physdev = physthing;
thing->data.logdev.visualizer = SDL_CreateTexture(state->renderers[0], SDL_PIXELFORMAT_RGBA8888, SDL_TEXTUREACCESS_TARGET, VISUALIZER_WIDTH, VISUALIZER_HEIGHT);
thing->data.logdev.postmix_lock = SDL_CreateMutex();
if (thing->data.logdev.visualizer) {
SDL_SetTextureBlendMode(thing->data.logdev.visualizer, SDL_BLENDMODE_BLEND);
}
thing->line_connected_to = physthing;
thing->ontick = LogicalDeviceThing_ontick;
thing->ondrag = DeviceThing_ondrag;
thing->ondrop = LogicalDeviceThing_ondrop;
thing->ondraw = LogicalDeviceThing_ondraw;
thing->can_be_dropped_onto = can_be_dropped_onto;
SetLogicalDeviceTitlebar(thing);