I'm implementing a floodfill function in C for the iPhone.
The fill works, although I'm having 2 issues.
The phone gives a memory warning after a few executions of the code below. Most likely a memory leak. Also note that the unsigned char *data (the image data) is being free()'d at the end of the floodfill.
(lesser issue) If I attempt to write RGB colors to pixels that are greater than approximately (r:200,g:200,b:200,a:200) I get weird artifacting happening. A workaround for this was to simply limit the values.
I suspect there may be a correlation between both of these problems.
The code below describes my flood-fill algorithm, using a stack:
.h:
typedef struct {
int red;
int green;
int blue;
int alpha;
} GUIColor;
struct pixel_st {
int x;
int y;
struct pixel_st *nextPixel;
};
typedef struct pixel_st pixel;
.m:
void floodFill(CGPoint location, GUIColor tc, GUIColor rc, size_t width, size_t height, unsigned char *data){
if (isGUIColorEqual(tc, rc)) return;
pixel* aPixel = (pixel *) malloc(sizeof (struct pixel_st));
NSLog(#"sizeof aPixel : %i",(int)sizeof(aPixel));
(*aPixel).x = location.x;
(*aPixel).y = location.y;
(*aPixel).nextPixel = NULL;
int i = 0;
NSLog(#"Replacement color A%i, R%i, G%i, B%i",rc.alpha,rc.red,rc.green, rc.blue);
while (aPixel != NULL){
pixel *oldPixel_p = aPixel;
pixel currentPixel = *aPixel;
aPixel = currentPixel.nextPixel;
//Now we do some boundary checks
if (!isOutOfBounds(currentPixel.x, currentPixel.y, width, height)){
//Grab the current Pixel color
GUIColor currentColor = getGUIColorFromPixelAtLocation(CGPointMake(currentPixel.x, currentPixel.y), width, height, data);
if (isGUIColorSimilar(currentColor, tc)){
//Colors are similar, lets continue the spread
setGUIColorToPixelAtLocation(CGPointMake(currentPixel.x, currentPixel.y), rc, width,height, data);
pixel *newPixel;
if ((newPixel = (pixel*) malloc(sizeof(struct pixel_st))) != NULL) {
(*newPixel).x = currentPixel.x;
(*newPixel).y = currentPixel.y-1;
(*newPixel).nextPixel = aPixel;
aPixel = newPixel;
}
if ((newPixel = (pixel*) malloc(sizeof(struct pixel_st))) != NULL) {
(*newPixel).x = currentPixel.x;
(*newPixel).y = currentPixel.y+1;
(*newPixel).nextPixel = aPixel;
aPixel = newPixel;
}
if ((newPixel = (pixel*) malloc(sizeof(struct pixel_st))) != NULL) {
(*newPixel).x = currentPixel.x+1;
(*newPixel).y = currentPixel.y;
(*newPixel).nextPixel = aPixel;
aPixel = newPixel;
}
if ((newPixel = (pixel*) malloc(sizeof(struct pixel_st))) != NULL) {
(*newPixel).x = currentPixel.x-1;
(*newPixel).y = currentPixel.y;
(*newPixel).nextPixel = aPixel;
aPixel = newPixel;
}
free(oldPixel_p);
i ++;
if (i == width * height * 4 * 5) break;
}
}
}
free(aPixel);
}
This implementation of the stack is based on the ObjFloodFill found here:
https://github.com/OgreSwamp/ObjFloodFill/blob/master/src/FloodFill.m
First of all, each if ((newPixel = (pixel*) malloc(... inside the loop allocates new memory block, so, you have 4 allocations inside the loop and only 1 deallocation.
Second, I can't understand why don't you simply use objects on stack? Do you really need to allocate newPixel, oldPixel and so on on the heap? Review the implementation, there might be much simpler way to implement the same also without managing the memory issues at all.
You need to move the deallocation of oldPixel_p to outside the if blocks, because it is "consumed" always.
Also, the final free only frees the first element in the list. The list may have more than one element. You need to step through the list and free all remaining elements.
Related
Hi I am trying to bind depth memory buffer but I get an error saying as below. I have no idea why this error is popping up.
The depth format is VK_FORMAT_D16_UNORM and the usage is VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT. I have read online that the TILING shouldnt be linear but then I get a different error. Thanks!!!
The code for creating and binding the image is as below.
VkImageCreateInfo imageInfo = {};
// If the depth format is undefined, use fallback as 16-byte value
if (Depth.format == VK_FORMAT_UNDEFINED) {
Depth.format = VK_FORMAT_D16_UNORM;
}
const VkFormat depthFormat = Depth.format;
VkFormatProperties props;
vkGetPhysicalDeviceFormatProperties(*deviceObj->gpu, depthFormat, &props);
if (props.linearTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) {
imageInfo.tiling = VK_IMAGE_TILING_LINEAR;
}
else if (props.optimalTilingFeatures & VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT) {
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
}
else {
std::cout << "Unsupported Depth Format, try other Depth formats.\n";
exit(-1);
}
imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
imageInfo.pNext = NULL;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = depthFormat;
imageInfo.extent.width = width;
imageInfo.extent.height = height;
imageInfo.extent.depth = 1;
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = NUM_SAMPLES;
imageInfo.queueFamilyIndexCount = 0;
imageInfo.pQueueFamilyIndices = NULL;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
imageInfo.flags = 0;
// User create image info and create the image objects
result = vkCreateImage(deviceObj->device, &imageInfo, NULL, &Depth.image);
assert(result == VK_SUCCESS);
// Get the image memory requirements
VkMemoryRequirements memRqrmnt;
vkGetImageMemoryRequirements(deviceObj->device, Depth.image, &memRqrmnt);
VkMemoryAllocateInfo memAlloc = {};
memAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
memAlloc.pNext = NULL;
memAlloc.allocationSize = 0;
memAlloc.memoryTypeIndex = 0;
memAlloc.allocationSize = memRqrmnt.size;
// Determine the type of memory required with the help of memory properties
pass = deviceObj->memoryTypeFromProperties(memRqrmnt.memoryTypeBits, 0, /* No requirements */ &memAlloc.memoryTypeIndex);
assert(pass);
// Allocate the memory for image objects
result = vkAllocateMemory(deviceObj->device, &memAlloc, NULL, &Depth.mem);
assert(result == VK_SUCCESS);
// Bind the allocated memeory
result = vkBindImageMemory(deviceObj->device, Depth.image, Depth.mem, 0);
assert(result == VK_SUCCESS);
Yes, linear tiling may not be supported for depth usage Images.
Consult the specification and Valid Usage section of VkImageCreateInfo. The capability is queried by vkGetPhysicalDeviceFormatProperties and vkGetPhysicalDeviceImageFormatProperties commands. Though depth formats are "opaque", so there is not much reason to use linear tiling.
This you seem to be doing in your code.
But the error informs you that you are trying to use a memory type that is not allowed for the given Image. Use vkGetImageMemoryRequirements command to query which memory types are allowed.
Possibly you have some error there (you are using 0x1 which is obviously not part of 0x84 per the message). You may want to reuse the example code in the Device Memory chapter of the specification. Provide your memoryTypeFromProperties implementation for more specific answer.
I accidentally set the typeIndex to 1 instead of i and it works now. In my defense I have been vulkan coding the whole day and my eyes are bleeding :). Thanks for the help.
bool VulkanDevice::memoryTypeFromProperties(uint32_t typeBits, VkFlags
requirementsMask, uint32_t *typeIndex)
{
// Search memtypes to find first index with those properties
for (uint32_t i = 0; i < 32; i++) {
if ((typeBits & 1) == 1) {
// Type is available, does it match user properties?
if ((memoryProperties.memoryTypes[i].propertyFlags & requirementsMask) == requirementsMask) {
*typeIndex = i;// was set to 1 :(
return true;
}
}
typeBits >>= 1;
}
// No memory types matched, return failure
return false;
}
I am checking if UIImage is darker or more whiter . I would like to use this method ,but only to check the third bottom part of the image ,not all of it .
I wonder how exactly to change it to check that,i am not that familiar with the pixels stuff .
BOOL isDarkImage(UIImage* inputImage){
BOOL isDark = FALSE;
CFDataRef imageData = CGDataProviderCopyData(CGImageGetDataProvider(inputImage.CGImage));
const UInt8 *pixels = CFDataGetBytePtr(imageData);
int darkPixels = 0;
long length = CFDataGetLength(imageData);
int const darkPixelThreshold = (inputImage.size.width*inputImage.size.height)*.25;
//should i change here the length ?
for(int i=0; i<length; i+=4)
{
int r = pixels[i];
int g = pixels[i+1];
int b = pixels[i+2];
//luminance calculation gives more weight to r and b for human eyes
float luminance = (0.299*r + 0.587*g + 0.114*b);
if (luminance<150) darkPixels ++;
}
if (darkPixels >= darkPixelThreshold)
isDark = YES;
I can just crop that part of the image, but this will be not efficient way, and wast time .
The solution marked correct here is a more thoughtful approach for getting the pixel data (more tolerant of differing formats) and also demonstrates how to address pixels. With a small adjustment, you can get the bottom of the image as follows:
+ (NSArray*)getRGBAsFromImage:(UIImage*)image
atX:(int)xx
andY:(int)yy
toX:(int)toX
toY:(int)toY {
// ...
int byteIndex = (bytesPerRow * yy) + xx * bytesPerPixel;
int byteIndexEnd = (bytesPerRow * toY) + toX * bytesPerPixel;
while (byteIndex < byteIndexEnd) {
// contents of the loop remain the same
// ...
}
To get the bottom third of the image, call this with xx=0, yy=2.0*image.height/3.0 and toX and toY equal to the image width and height, respectively. Loop the colors in the returned array and compute luminance as your post suggests.
It seems like this should be simpler than I'm finding it to be.
I have an AVFoundation frame coming back in the standard delegate method:
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
where I would like to convert the frame to greyscale using the Accelerate.Framework.
There is a family of conversion methods in the framework, including vImageConvert_RGBA8888toPlanar8(), which looks like it might be what I would like to see, however, I can't find any examples of how to use them!
So far, I have the code:
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
{
#autoreleasepool {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
/*Lock the image buffer*/
CVPixelBufferLockBaseAddress(imageBuffer,0);
/*Get information about the image*/
uint8_t *baseAddress = (uint8_t *)CVPixelBufferGetBaseAddress(imageBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
size_t stride = CVPixelBufferGetBytesPerRow(imageBuffer);
// vImage In
Pixel_8 *bitmap = (Pixel_8 *)malloc(width * height * sizeof(Pixel_8));
const vImage_Buffer inImage = { bitmap, height, width, stride };
//How can I take this inImage and convert it to greyscale?????
//vImageConvert_RGBA8888toPlanar8()??? Is the correct starting format here??
}
}
So I have two questions:
(1) In the code above, is RBGA8888 the correct starting format?
(2) How can I actually make the Accelerate.Framework call to convert to greyscale?
There is an easier option here. If you change the camera acquire format to YUV, then you already have a greyscale frame that you can use as you like. When setting up your data output, use something like:
dataOutput.videoSettings = #{ (id)kCVPixelBufferPixelFormatTypeKey : #(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) };
You can then access the Y plane in your capture callback using:
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
uint8_t *yPlane = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
... do stuff with your greyscale camera image ...
CVPixelBufferUnlockBaseAddress(pixelBuffer);
The vImage method is to use vImageMatrixMultiply_Planar8 and a 1x3 matrix.
vImageConvert_RGBA8888toPlanar8 is the function you use to convert a RGBA8888 buffer into 4 planar buffers. These are used by vImageMatrixMultiply_Planar8. vImageMatrixMultiply_ARGB8888 will do it too in one pass, but your gray channel will be interleaved with three other channels in the result. vImageConvert_RGBA8888toPlanar8 itself doesn't do any math. All it does is separate your interleaved image into separate image planes.
If you need to adjust the gamma as well, then probably vImageConvert_AnyToAny() is the easy choice. It will do the fully color managed conversion from your RGB format to a grayscale colorspace. See vImage_Utilities.h.
I like Tarks answer better though. It just leaves you in a position of having to color manage the Luminance manually (if you care).
Convert BGRA Image to Grayscale with Accelerate vImage
This method is meant to illustrate getting Accelerate's vImage use in converting BGR images to grayscale. Your image may very well be in RGBA format and you'll need to adjust the matrix accordingly, but the camera outputs BGRA so I'm using it here. The values in the matrix are the same values used in OpenCV for cvtColor, there are other values you might play with like luminosity. I assume you malloc the appropriate amount of memory for the result. In the case of grayscale it is only 1-channel or 1/4 the memory used for BGRA. If anyone finds issues with this code please leave a comment.
Performance note
Converting to grayscale in this way may NOT be the fastest. You should check the performance of any method in your environment. Brad Larson's GPUImage might be faster, or even OpenCV's cvtColor. In any case you will want to remove the calls to malloc and free for the intermediate buffers and manage them for the app lifecycle. Otherwise, the function call will be dominated by the malloc and free. Apple's docs recommend reusing the whole vImage_Buffer when possible.
You can also read about solving the same problem with NEON intrinsics.
Finally, the fastest method is not converting at all. If you're getting image data from the device camera the device camera is natively in the kCVPixelFormatType_420YpCbCr8BiPlanarFullRange format. Meaning, grabbing the first plane's data (Y-Channel, luma) is the fastest way to get grayscale.
BGRA to Grayscale
- (void)convertBGRAFrame:(const CLPBasicVideoFrame &)bgraFrame toGrayscale:(CLPBasicVideoFrame &)grayscaleFrame
{
vImage_Buffer bgraImageBuffer = {
.width = bgraFrame.width,
.height = bgraFrame.height,
.rowBytes = bgraFrame.bytesPerRow,
.data = bgraFrame.rawPixelData
};
void *intermediateBuffer = malloc(bgraFrame.totalBytes);
vImage_Buffer intermediateImageBuffer = {
.width = bgraFrame.width,
.height = bgraFrame.height,
.rowBytes = bgraFrame.bytesPerRow,
.data = intermediateBuffer
};
int32_t divisor = 256;
// int16_t a = (int16_t)roundf(1.0f * divisor);
int16_t r = (int16_t)roundf(0.299f * divisor);
int16_t g = (int16_t)roundf(0.587f * divisor);
int16_t b = (int16_t)roundf(0.114f * divisor);
const int16_t bgrToGray[4 * 4] = { b, 0, 0, 0,
g, 0, 0, 0,
r, 0, 0, 0,
0, 0, 0, 0 };
vImage_Error error;
error = vImageMatrixMultiply_ARGB8888(&bgraImageBuffer, &intermediateImageBuffer, bgrToGray, divisor, NULL, NULL, kvImageNoFlags);
if (error != kvImageNoError) {
NSLog(#"%s, vImage error %zd", __PRETTY_FUNCTION__, error);
}
vImage_Buffer grayscaleImageBuffer = {
.width = grayscaleFrame.width,
.height = grayscaleFrame.height,
.rowBytes = grayscaleFrame.bytesPerRow,
.data = grayscaleFrame.rawPixelData
};
void *scratchBuffer = malloc(grayscaleFrame.totalBytes);
vImage_Buffer scratchImageBuffer = {
.width = grayscaleFrame.width,
.height = grayscaleFrame.height,
.rowBytes = grayscaleFrame.bytesPerRow,
.data = scratchBuffer
};
error = vImageConvert_ARGB8888toPlanar8(&intermediateImageBuffer, &grayscaleImageBuffer, &scratchImageBuffer, &scratchImageBuffer, &scratchImageBuffer, kvImageNoFlags);
if (error != kvImageNoError) {
NSLog(#"%s, vImage error %zd", __PRETTY_FUNCTION__, error);
}
free(intermediateBuffer);
free(scratchBuffer);
}
CLPBasicVideoFrame.h - For reference
typedef struct
{
size_t width;
size_t height;
size_t bytesPerRow;
size_t totalBytes;
unsigned long pixelFormat;
void *rawPixelData;
} CLPBasicVideoFrame;
I got through the grayscale conversion, but was having trouble with the quality when I found this book on the web called Instant OpenCV for iOS. I personally picked up a copy and it has a number of gems, although the code is bit of a mess. On the bright-side it is a very reasonably priced eBook.
I'm very curious about that matrix. I toyed around with it for hours trying to figure out what the arrangement should be. I would have thought the values should be on the diagonal, but the Instant OpenCV guys put it as above.
if you need to use BGRA vide streams - you can use this excellent conversion
here
This is the function you'll need to take:
void neon_convert (uint8_t * __restrict dest, uint8_t * __restrict src, int numPixels)
{
int i;
uint8x8_t rfac = vdup_n_u8 (77);
uint8x8_t gfac = vdup_n_u8 (151);
uint8x8_t bfac = vdup_n_u8 (28);
int n = numPixels / 8;
// Convert per eight pixels
for (i=0; i < n; ++i)
{
uint16x8_t temp;
uint8x8x4_t rgb = vld4_u8 (src);
uint8x8_t result;
temp = vmull_u8 (rgb.val[0], bfac);
temp = vmlal_u8 (temp,rgb.val[1], gfac);
temp = vmlal_u8 (temp,rgb.val[2], rfac);
result = vshrn_n_u16 (temp, 8);
vst1_u8 (dest, result);
src += 8*4;
dest += 8;
}
}
more optimisations (using assembly) are in the link
(1) My experience with the iOS camera framework has been with images in the kCMPixelFormat_32BGRA format, which is compatible with the ARGB8888 family of functions. (It may be possible to use other formats as well.)
(2) The simplest way to convert from BGR to grayscale on iOS is to use vImageMatrixMultiply_ARGB8888ToPlanar8():
https://developer.apple.com/documentation/accelerate/1546979-vimagematrixmultiply_argb8888top
Here is a fairly complete example written in Swift. I'm assuming the Objective-C code would be similar.
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
// TODO: report error
return
}
// Lock the image buffer
if (kCVReturnSuccess != CVPixelBufferLockBaseAddress(imageBuffer, CVPixelBufferLockFlags.readOnly)) {
// TODO: report error
return
}
defer {
CVPixelBufferUnlockBaseAddress(imageBuffer, CVPixelBufferLockFlags.readOnly)
}
// Create input vImage_Buffer
let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer)
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
let stride = CVPixelBufferGetBytesPerRow(imageBuffer)
var inImage = vImage_Buffer(data: baseAddress, height: UInt(height), width: UInt(width), rowBytes: stride)
// Create output vImage_Buffer
let bitmap = malloc(width * height)
var outImage = vImage_Buffer(data: bitmap, height: UInt(height), width: UInt(width), rowBytes: width)
defer {
// Make sure to free unless the caller is responsible for this
free(bitmap)
}
// Arbitrary divisor to scale coefficients to integer values
let divisor: Int32 = 0x1000
let fDivisor = Float(divisor)
// Rec.709 coefficients
var coefficientsMatrix = [
Int16(0.0722 * fDivisor), // blue
Int16(0.7152 * fDivisor), // green
Int16(0.2126 * fDivisor), // red
0 // alpha
]
// Convert to greyscale
if (kvImageNoError != vImageMatrixMultiply_ARGB8888ToPlanar8(
&inImage, &outImage, &coefficientsMatrix, divisor, nil, 0, vImage_Flags(kvImageNoFlags))) {
// TODO: report error
return
}
The code above was inspired by a tutorial from Apple on grayscale conversion, which can be found at the following link. It also includes conversion to a CGImage if that is needed. Note that they assume RGB order instead of BGR, and they only provide a 3 coefficients instead of 4 (mistake?)
https://developer.apple.com/documentation/accelerate/vimage/converting_color_images_to_grayscale
I am trying to do a simple playback from a file functionality and it appears that my callback function is never called. It doesn't really make sense because all of the OSStatuses come back 0 and other numbers all appear correct as well (like the output packets read pointer from AudioFileReadPackets).
Here is the setup:
OSStatus stat;
stat = AudioFileOpenURL(
(CFURLRef)urlpath, kAudioFileReadPermission, 0, &aStreamData->aFile
);
UInt32 dsze = 0;
stat = AudioFileGetPropertyInfo(
aStreamData->aFile, kAudioFilePropertyDataFormat, &dsze, 0
);
stat = AudioFileGetProperty(
aStreamData->aFile, kAudioFilePropertyDataFormat, &dsze, &aStreamData->aDescription
);
stat = AudioQueueNewOutput(
&aStreamData->aDescription, bufferCallback, aStreamData, NULL, NULL, 0, &aStreamData->aQueue
);
aStreamData->pOffset = 0;
for(int i = 0; i < NUM_BUFFERS; i++) {
stat = AudioQueueAllocateBuffer(
aStreamData->aQueue, aStreamData->aDescription.mBytesPerPacket, &aStreamData->aBuffer[i]
);
bufferCallback(aStreamData, aStreamData->aQueue, aStreamData->aBuffer[i]);
}
stat = AudioQueuePrime(aStreamData->aQueue, 0, NULL);
stat = AudioQueueStart(aStreamData->aQueue, NULL);
(Not shown is where I'm checking the value of stat in between the functions, it just comes back normal.)
And the callback function:
void bufferCallback(void *uData, AudioQueueRef queue, AudioQueueBufferRef buffer) {
UInt32 bread = 0;
UInt32 pread = buffer->mAudioDataBytesCapacity / player->aStreamData->aDescription.mBytesPerPacket;
OSStatus stat;
stat = AudioFileReadPackets(
player->aStreamData->aFile, false, &bread, NULL, player->aStreamData->pOffset, &pread, buffer->mAudioData
);
buffer->mAudioDataByteSize = bread;
stat = AudioQueueEnqueueBuffer(queue, buffer, 0, NULL);
player->aStreamData->pOffset += pread;
}
Where aStreamData is my user data struct (typedefed so I can use it as a class property) and player is a static instance of the controlling Objective-C class. If any other code is wanted please let me know. I am a bit at my wit's end. Printing any of the numbers involved here yields the correct result, including functions in bufferCallback when I call it myself in the allocate loop. It just never gets called thereafter. The start up method returns and nothing happens.
Also anecdotally, I am using a peripheral device (an MBox Pro 3) to play the sound which CoreAudio only boots up when it is about to output. IE if I start iTunes or something, the speakers pop faintly and there is an LED that goes from blinking to solid. The device boots up like it does so CA is definitely doing something. (Also I've of course tried it with the onboard Macbook sound sans the device.)
I've read other solutions to problems that sound similiar and they don't work. Stuff like using multiple buffers which I am doing now and doesn't appear to make any difference.
I basically assume I am doing something obviously wrong somehow but not sure what it could be. I've read the relevant documentation, looked at the available code examples and scoured the net a bit for answers and it appears that this is all I need to do and it should just go.
At the very least, is there anything else I can do to investigate?
My first answer was not good enough, so I compiled a minimal example that will play a 2 channel, 16 bit wave file.
The main difference from your code is that I made a property listener listening for play start and stop events.
As for your code, it seems legit at first glance. Two things I will point out, though:
1. Is seems you are allocating buffers with TOO SMALL a buffer size. I have noticed that AudioQueues won't play if the buffers are too small, which seems to fit your problem.
2. Have you verified the properties returned?
Back to my code example:
Everything is hard coded, so it is not exactly good coding practice, but it shows how you can do it.
AudioStreamTest.h
#import <Foundation/Foundation.h>
#import <AudioToolbox/AudioToolbox.h>
uint32_t bufferSizeInSamples;
AudioFileID file;
UInt32 currentPacket;
AudioQueueRef audioQueue;
AudioQueueBufferRef buffer[3];
AudioStreamBasicDescription audioStreamBasicDescription;
#interface AudioStreamTest : NSObject
- (void)start;
- (void)stop;
#end
AudioStreamTest.m
#import "AudioStreamTest.h"
#implementation AudioStreamTest
- (id)init
{
self = [super init];
if (self) {
bufferSizeInSamples = 441;
file = NULL;
currentPacket = 0;
audioStreamBasicDescription.mBitsPerChannel = 16;
audioStreamBasicDescription.mBytesPerFrame = 4;
audioStreamBasicDescription.mBytesPerPacket = 4;
audioStreamBasicDescription.mChannelsPerFrame = 2;
audioStreamBasicDescription.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
audioStreamBasicDescription.mFormatID = kAudioFormatLinearPCM;
audioStreamBasicDescription.mFramesPerPacket = 1;
audioStreamBasicDescription.mReserved = 0;
audioStreamBasicDescription.mSampleRate = 44100;
}
return self;
}
- (void)start {
AudioQueueNewOutput(&audioStreamBasicDescription, AudioEngineOutputBufferCallback, (__bridge void *)(self), NULL, NULL, 0, &audioQueue);
AudioQueueAddPropertyListener(audioQueue, kAudioQueueProperty_IsRunning, AudioEnginePropertyListenerProc, NULL);
AudioQueueStart(audioQueue, NULL);
}
- (void)stop {
AudioQueueStop(audioQueue, YES);
AudioQueueRemovePropertyListener(audioQueue, kAudioQueueProperty_IsRunning, AudioEnginePropertyListenerProc, NULL);
}
void AudioEngineOutputBufferCallback(void *inUserData, AudioQueueRef inAQ, AudioQueueBufferRef inBuffer) {
if (file == NULL) return;
UInt32 bytesRead = bufferSizeInSamples * 4;
UInt32 packetsRead = bufferSizeInSamples;
AudioFileReadPacketData(file, false, &bytesRead, NULL, currentPacket, &packetsRead, inBuffer->mAudioData);
inBuffer->mAudioDataByteSize = bytesRead;
currentPacket += packetsRead;
if (bytesRead == 0) {
AudioQueueStop(inAQ, false);
}
else {
AudioQueueEnqueueBuffer(inAQ, inBuffer, 0, NULL);
}
}
void AudioEnginePropertyListenerProc (void *inUserData, AudioQueueRef inAQ, AudioQueuePropertyID inID) {
//We are only interested in the property kAudioQueueProperty_IsRunning
if (inID != kAudioQueueProperty_IsRunning) return;
//Get the status of the property
UInt32 isRunning = false;
UInt32 size = sizeof(isRunning);
AudioQueueGetProperty(inAQ, kAudioQueueProperty_IsRunning, &isRunning, &size);
if (isRunning) {
currentPacket = 0;
NSString *fileName = #"/Users/roy/Documents/XCodeProjectsData/FUZZ/03.wav";
NSURL *fileURL = [[NSURL alloc] initFileURLWithPath: fileName];
AudioFileOpenURL((__bridge CFURLRef) fileURL, kAudioFileReadPermission, 0, &file);
for (int i = 0; i < 3; i++){
AudioQueueAllocateBuffer(audioQueue, bufferSizeInSamples * 4, &buffer[i]);
UInt32 bytesRead = bufferSizeInSamples * 4;
UInt32 packetsRead = bufferSizeInSamples;
AudioFileReadPacketData(file, false, &bytesRead, NULL, currentPacket, &packetsRead, buffer[i]->mAudioData);
buffer[i]->mAudioDataByteSize = bytesRead;
currentPacket += packetsRead;
AudioQueueEnqueueBuffer(audioQueue, buffer[i], 0, NULL);
}
}
else {
if (file != NULL) {
AudioFileClose(file);
file = NULL;
for (int i = 0; i < 3; i++) {
AudioQueueFreeBuffer(audioQueue, buffer[i]);
buffer[i] = NULL;
}
}
}
}
-(void)dealloc {
[super dealloc];
AudioQueueDispose(audioQueue, true);
audioQueue = NULL;
}
#end
Lastly, I want to include some research I have done today to test the robustness of AudioQueues.
I have noticed that if you make too small AudioQueue buffers, it won't play at all. That made me play around a bit to see why it is not playing.
If I try buffer size that can hold only 150 samples, I get no sound at all.
If I try buffer size that can hold 175 samples, it plays the whole song through, but with A lot of distortion. 175 amounts to a tad less than 4 ms of audio.
AudioQueue keeps asking for new buffers as long as you keep supplying buffers. That is regardless of AudioQueue actually playing your buffers or not.
If you supply a buffer with size 0, the buffer will be lost and an error kAudioQueueErr_BufferEmpty is returned for that queue enqueue request. You will never see AudioQueue ask you to fill that buffer again. If this happened for the last queue you have posted, AudioQueue will stop asking you to fill any more buffers. In that case you will not hear any more audio for that session.
To see why AudioQueues is not playing anything with smaller buffer sizes, I made a test to see if my callback is called at all even when there is no sound. The answer is that the buffers gets called all the time as long as AudioQueues is playing and needs data.
So if you keep feeding buffers to the queue, no buffer is ever lost. It doesn't happen. Unless there is an error, of course.
So why is no sound playing?
I tested to see if 'AudioQueueEnqueueBuffer()' returned any errors. It did not. No other errors within my play routine either. The data returned from reading from file is also good.
Everything is normal, buffers are good, data re-enqueued is good, there is just no sound.
So my last test was to slowly increase buffer size till I could hear anything. I finally heard faint and sporadic distortion.
Then it came to me...
It seems that the problem lies with that the system tries to keep the stream in sync with time so if you enqueue audio, and the time for the audio you wanted to play has passed, it will just skip that part of the buffer. If the buffer size becomes too small, more and more data is dropped or skipped until the audio system is in sync again. Which is never if the buffer size is too small. (You can hear this as distortion if you chose a buffer size that is barely large enough to support continuous play.)
If you think about it, it is the only way the audio queue can work, but it is a good realisation when you are clueless like me and "discover" how it really works.
I decided to take a look at this again and was able to solve it by making the buffers larger. I've accepted the answer by #RoyGal since it was their suggestion but I wanted to provide the actual code that works since I guess others are having the same problem (question has a few favorites that aren't me at the moment).
One thing I tried was making the packet size larger:
aData->aDescription.mFramesPerPacket = 512; // or some other number
aData->aDescription.mBytesPerPacket = (
aData->aDescription.mFramesPerPacket * aData->aDescription.mBytesPerFrame
);
This does NOT work: it causes AudioQueuePrime to fail with an AudioConverterNew returned -50 message. I guess it wants mFramesPerPacket to be 1 for PCM.
(I also tried setting the kAudioQueueProperty_DecodeBufferSizeFrames property which didn't seem to do anything. Not sure what it's for.)
The solution seems to be to only allocate the buffer(s) with the specified size:
AudioQueueAllocateBuffer(
aData->aQueue,
aData->aDescription.mBytesPerPacket * N_BUFFER_PACKETS / N_BUFFERS,
&aData->aBuffer[i]
);
And the size has to be sufficiently large. I found the magic number is:
mBytesPerPacket * 1024 / N_BUFFERS
(Where N_BUFFERS is the number of buffers and should be > 1 or playback is choppy.)
Here is an MCVE demonstrating the issue and solution:
#import <Foundation/Foundation.h>
#import <AudioToolbox/AudioToolbox.h>
#import <AudioToolbox/AudioQueue.h>
#import <AudioToolbox/AudioFile.h>
#define N_BUFFERS 2
#define N_BUFFER_PACKETS 1024
typedef struct AStreamData {
AudioFileID aFile;
AudioQueueRef aQueue;
AudioQueueBufferRef aBuffer[N_BUFFERS];
AudioStreamBasicDescription aDescription;
SInt64 pOffset;
volatile BOOL isRunning;
} AStreamData;
void printASBD(AudioStreamBasicDescription* desc) {
printf("mSampleRate = %d\n", (int)desc->mSampleRate);
printf("mBytesPerPacket = %d\n", desc->mBytesPerPacket);
printf("mFramesPerPacket = %d\n", desc->mFramesPerPacket);
printf("mBytesPerFrame = %d\n", desc->mBytesPerFrame);
printf("mChannelsPerFrame = %d\n", desc->mChannelsPerFrame);
printf("mBitsPerChannel = %d\n", desc->mBitsPerChannel);
}
void bufferCallback(
void *vData, AudioQueueRef aQueue, AudioQueueBufferRef aBuffer
) {
AStreamData* aData = (AStreamData*)vData;
UInt32 bRead = 0;
UInt32 pRead = (
aBuffer->mAudioDataBytesCapacity / aData->aDescription.mBytesPerPacket
);
OSStatus stat;
stat = AudioFileReadPackets(
aData->aFile, false, &bRead, NULL, aData->pOffset, &pRead, aBuffer->mAudioData
);
if(stat != 0) {
printf("AudioFileReadPackets returned %d\n", stat);
}
if(pRead == 0) {
aData->isRunning = NO;
return;
}
aBuffer->mAudioDataByteSize = bRead;
stat = AudioQueueEnqueueBuffer(aQueue, aBuffer, 0, NULL);
if(stat != 0) {
printf("AudioQueueEnqueueBuffer returned %d\n", stat);
}
aData->pOffset += pRead;
}
AStreamData* beginPlayback(NSURL* path) {
static AStreamData* aData;
aData = malloc(sizeof(AStreamData));
OSStatus stat;
stat = AudioFileOpenURL(
(CFURLRef)path, kAudioFileReadPermission, 0, &aData->aFile
);
printf("AudioFileOpenURL returned %d\n", stat);
UInt32 dSize = 0;
stat = AudioFileGetPropertyInfo(
aData->aFile, kAudioFilePropertyDataFormat, &dSize, 0
);
printf("AudioFileGetPropertyInfo returned %d\n", stat);
stat = AudioFileGetProperty(
aData->aFile, kAudioFilePropertyDataFormat, &dSize, &aData->aDescription
);
printf("AudioFileGetProperty returned %d\n", stat);
printASBD(&aData->aDescription);
stat = AudioQueueNewOutput(
&aData->aDescription, bufferCallback, aData, NULL, NULL, 0, &aData->aQueue
);
printf("AudioQueueNewOutput returned %d\n", stat);
aData->pOffset = 0;
for(int i = 0; i < N_BUFFERS; i++) {
// change YES to NO for stale playback
if(YES) {
stat = AudioQueueAllocateBuffer(
aData->aQueue,
aData->aDescription.mBytesPerPacket * N_BUFFER_PACKETS / N_BUFFERS,
&aData->aBuffer[i]
);
} else {
stat = AudioQueueAllocateBuffer(
aData->aQueue,
aData->aDescription.mBytesPerPacket,
&aData->aBuffer[i]
);
}
printf(
"AudioQueueAllocateBuffer returned %d for aBuffer[%d] with capacity %d\n",
stat, i, aData->aBuffer[i]->mAudioDataBytesCapacity
);
bufferCallback(aData, aData->aQueue, aData->aBuffer[i]);
}
UInt32 numFramesPrepared = 0;
stat = AudioQueuePrime(aData->aQueue, 0, &numFramesPrepared);
printf("AudioQueuePrime returned %d with %d frames prepared\n", stat, numFramesPrepared);
stat = AudioQueueStart(aData->aQueue, NULL);
printf("AudioQueueStart returned %d\n", stat);
UInt32 pSize = sizeof(UInt32);
UInt32 isRunning;
stat = AudioQueueGetProperty(
aData->aQueue, kAudioQueueProperty_IsRunning, &isRunning, &pSize
);
printf("AudioQueueGetProperty returned %d\n", stat);
aData->isRunning = !!isRunning;
return aData;
}
void endPlayback(AStreamData* aData) {
OSStatus stat = AudioQueueStop(aData->aQueue, NO);
printf("AudioQueueStop returned %d\n", stat);
}
NSString* getPath() {
// change NO to YES and enter path to hard code
if(NO) {
return #"";
}
char input[512];
printf("Enter file path: ");
scanf("%[^\n]", input);
return [[NSString alloc] initWithCString:input encoding:NSASCIIStringEncoding];
}
int main(int argc, const char* argv[]) {
NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];
NSURL* path = [NSURL fileURLWithPath:getPath()];
AStreamData* aData = beginPlayback(path);
if(aData->isRunning) {
do {
printf("Queue is running...\n");
[NSThread sleepForTimeInterval:1.0];
} while(aData->isRunning);
endPlayback(aData);
} else {
printf("Playback did not start\n");
}
[pool drain];
return 0;
}
I am trying to increase the performance of the update(); function below. The numbers inside the mathNumber variable will come from an NSString created from a text field. Even though I'm using five numbers I would like it to be able to run any amount that the user inserts into a text field. What are some ways I could speed up the code in update(); with C and/or Objective-C? I also would like it to work on the Mac and iPhone.
typedef struct {
float *left;
float *right;
float *equals;
int operation;
} MathVariable;
#define MULTIPLY 1
#define DIVIDE 2
#define ADD 3
#define SUBTRACT 4
MathVariable *mathVariable;
float *mathPointer;
float newNumber;
void init();
void update();
float solution(float *left, float *right, int *operation);
void init()
{
float *mathNumber = (float *) malloc(sizeof(float) * 9);
mathNumber[0] =-1.0;
mathNumber[1] =-2.0;
mathNumber[2] = 3.0;
mathNumber[3] = 4.0;
mathNumber[4] = 5.0;
mathNumber[5] = 0.0;
mathNumber[6] = 0.0;
mathNumber[7] = 0.0;
mathNumber[8] = 0.0;
mathVariable = (MathVariable *) malloc(sizeof(MathVariable) * 4);
mathVariable[0].equals = &mathPointer[5];
mathVariable[0].left = &mathPointer[2];
mathVariable[0].operation = MULTIPLY;
mathVariable[0].right = &mathPointer[3];
mathVariable[1].equals = &mathPointer[6];
mathVariable[1].left = &mathPointer[1];
mathVariable[1].operation = SUBTRACT;
mathVariable[1].right = &mathPointer[5];
mathVariable[2].equals = &mathPointer[7];
mathVariable[2].left = &mathPointer[0];
mathVariable[2].operation = ADD;
mathVariable[2].right = &mathPointer[6];
mathVariable[3].equals = &mathPointer[8];
mathVariable[3].left = &mathPointer[7];
mathVariable[3].operation = MULTIPLY;
mathVariable[3].right = &mathPointer[4];
return self;
}
// This is updated with a timer
void update()
{
int i;
for (i = 0; i < 4; i++)
{
*mathVariable[i].equals = solution(mathVariable[i].left, mathVariable[i].right, &mathVariable[i].operation);
}
// Below is the equivalent of: newNumber = (-1.0 + (-2.0 - 3.0 * 4.0)) * 5.0;
// newNumber should equal -75
newNumber = mathPointer[8];
}
float solution(float *left, float *right, int *operation)
{
if ((*operation) == MULTIPLY)
{
return (*left) * (*right);
}
else if ((*operation) == DIVIDE)
{
return (*left) / (*right);
}
else if ((*operation) == ADD)
{
return (*left) + (*right);
}
else if ((*operation) == SUBTRACT)
{
return (*left) - (*right);
}
else
{
return 0.0;
}
}
EDIT:
I first must say thank you for all of your kind posts. This is the first forum I've gotten people that don't tell me I'm a complete idiot. Sorry about the return self; I didn't realize this was an objective-C forum too (thus why I hastily used C). I have my own parser which is slow but I'm not concerned with its speed. All I want is to speed up the update() function since it slows everything down and 90% of the objects use it. Also, I'm try to get it to work faster with iOS devices since I can't compile anything in the text boxes. If you have any other advice on making update() faster I thank you.
Thanks again,
Jonathan
EDIT 2:
Well I got it to run faster by changing it from:
int i;
for (i = 0; i < 4; i++)
{
*mathVariable[i].equals = solution(*mathVariable[i].left, *mathVariable[i].right, mathVariable[i].operation);
}
To:
*mathVariable[0].equals = solution(*mathVariable[0].left, *mathVariable[0].right, mathVariable[0].operation);
*mathVariable[1].equals = solution(*mathVariable[1].left, *mathVariable[1].right, mathVariable[1].operation);
*mathVariable[2].equals = solution(*mathVariable[2].left, *mathVariable[2].right, mathVariable[2].operation);
*mathVariable[3].equals = solution(*mathVariable[3].left, *mathVariable[3].right, mathVariable[3].operation);
Is there any other way to increment it as fast as the preloaded numbers in the array like above?
Your code is a mix of styles, and contains some unwarranted uses of pointers (e.g. when passing operation to solution). It is unclear why you are passing the floats by reference, but maybe you intend that these change be changed and the expression reevaluated?
Below are some changes both to tidy and incidentally speed it up - the cost of any of this is not high and you may be guilt of premature optimization. As #Dave commented there are libraries to do parsing for you, but if you're targeting simple math expressions an operator precedence stack-based parser/evaluator is easy enough to code.
Suggestion 1: use enum - cleaner:
typedef enum { MULTIPLY, DIVIDE, ADD, SUBTRACT } BinaryOp;
typedef struct
{
float *left;
float *right;
float *equals;
BinaryOp operation;
} MathVariable;
Suggestion 2: use switch - cleaner and probably faster as well:
float solution(float left, float right, int operation)
{
switch(operation)
{
case MULTIPLY:
return left * right;
case DIVIDE:
return left / right;
case ADD:
return left + right;
case SUBTRACT:
return left - right;
default:
return 0.0;
}
}
Note I also removed passing pointers, the call is now:
*mathVariable[i].equals = solution(*mathVariable[i].left,
*mathVariable[i].right,
mathVariable[i].operation);
Now an OO person will probably object (:-)) to the switch (or the if/else) and argue each node (your MathVariable) should be an instance which knows how to perform its own operation. A C person might suggest you use function pointers in the node so they can perform their own operation. All this is design and you'll have to figure that out yourself.