How do I avoid app termination due to memory issue? - objective-c

Question

I am fetching 700.000 rows from a web service and my app is crashing after a while being terminated due to memory issue, it consumes about 1 GB of ram before it crashes, the code is fairly simple, I fetch the JSON, I put into an array, I loop the array and insert into core data and once done I save the context
the code is shown below
+ (void)fetchTillDataAll:(int)tillId :(int)startAtRow :(int)takeNoOfRows {
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillDataAll:tillId = %d, startAtRow = %d, takeNoOfRows = %d", tillId, startAtRow, takeNoOfRows);
}
NSString *finalURL = [NSString stringWithFormat:#"https://host.domain.com:5443/api/till/tilldatav2/%d?StartAtRow=%d&TakeNoOfRows=%d",tillId, startAtRow, takeNoOfRows];
[[[NSURLSession sharedSession] dataTaskWithURL:[NSURL URLWithString:finalURL]
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
if (error != nil) {
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillDataAll:Transport error %#", error);
}
} else {
NSHTTPURLResponse *responseHTTP;
responseHTTP = (NSHTTPURLResponse *) response;
if(responseHTTP.statusCode != 200) {
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillDataAll:Server Error %d", (int) responseHTTP.statusCode);
}
} else {
NSArray *tillBasicDataArray = [NSJSONSerialization JSONObjectWithData:data
options:0
error:NULL];
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillDataAll:tillBasicDataArray count = %lu", (unsigned long)[tillBasicDataArray count]);
NSLog(#"WebServices:fetchTillDataAll:tillBasicDataArray looks like %#",tillBasicDataArray);
}
AppDelegate *appDelegate = (AppDelegate *)[[UIApplication sharedApplication]delegate];
//NSManagedObjectContext *context =
//appDelegate.persistentContainer.viewContext;
//context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSPersistentContainer *container = appDelegate.persistentContainer;
[container performBackgroundTask:^(NSManagedObjectContext *context ) {
context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSDictionary *tillBasicDataDict = Nil;
//Loop through the array and for each dictionary insert into local DB
// lets work on concurrency here
for (id element in tillBasicDataArray){
tillBasicDataDict = element;
NSString *itemId = [tillBasicDataDict objectForKey:#"itemId"];
NSString *brandId = [tillBasicDataDict objectForKey:#"companyId"];
NSString *languageId = [tillBasicDataDict objectForKey:#"languageCode"];
NSString *colorCode = [NSString stringWithFormat:#"%#", [tillBasicDataDict objectForKey:#"colorCode"]];
NSString *discountable = [tillBasicDataDict objectForKey:#"discountable"];
NSString *exchangeable = [tillBasicDataDict objectForKey:#"exchangeable"];
NSString *noos14 = [tillBasicDataDict objectForKey:#"noos14"];
NSString *sizeCode = [NSString stringWithFormat:#"%#", [tillBasicDataDict objectForKey:#"sizeCode"]];
NSString *taxGroup = [tillBasicDataDict objectForKey:#"taxGroupId"];
NSString *taxRegion = [tillBasicDataDict objectForKey:#"taxRegion"];
NSString *tradeItemDesc = [tillBasicDataDict objectForKey:#"tradeItemDesc"];
NSString *withTax = [tillBasicDataDict objectForKey:#"withTax"];
NSString *status = [tillBasicDataDict objectForKey:#"status"];
// Use Core Data FMD
NSManagedObject *newPimItem = Nil;
newPimItem = [NSEntityDescription
insertNewObjectForEntityForName:#"TillData"
inManagedObjectContext:context];
[newPimItem setValue:itemId forKey:#"itemId"];
[newPimItem setValue:brandId forKey:#"brandId"];
[newPimItem setValue:languageId forKey:#"languageCode"];
[newPimItem setValue:colorCode forKey:#"colorCode"];
[newPimItem setValue:discountable forKey:#"discountable"];
[newPimItem setValue:exchangeable forKey:#"exchangeable"];
[newPimItem setValue:noos14 forKey:#"noos14"];
[newPimItem setValue:sizeCode forKey:#"sizeCode"];
[newPimItem setValue:[NSNumber numberWithInt:[taxGroup intValue]] forKey:#"taxGroup"];
[newPimItem setValue:taxRegion forKey:#"taxRegion"];
[newPimItem setValue:tradeItemDesc forKey:#"tradeItemDesc"];
[newPimItem setValue:[NSNumber numberWithInt:[withTax intValue]] forKey:#"withTax"];
[newPimItem setValue:[NSNumber numberWithInt:[status intValue]] forKey:#"status"];
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillDataAll:ItemId in loop = %#", itemId);
NSLog(#"WebServices:fetchTillDataAll:newPimItem = %#", newPimItem);
NSLog(#"WebServices:fetchTillDataAll:CoreData error = %#", error);
}
}
NSError *error = nil;
if (![context save:&error]) {
NSLog(#"Failure to save context: %#\n%#", [error localizedDescription], [error userInfo]);
abort();
} else {
NSUserDefaults *tillUserDefaults = [NSUserDefaults standardUserDefaults];
[tillUserDefaults setInteger:1 forKey:#"hasTillData"];
[tillUserDefaults synchronize];
}
}];
}
}
}] resume];
}
What can I do minimize the foot print so that I am able to download the data? I absolutely must have the data locally in order to allow offline capabilities in the app
----- EDIT -----
After implementing splitting the NSArray into an array of array I still get the same problem, below if the new code as suggested:
split method
+ (NSArray *) splitIntoArraysOfBatchSize:(NSArray *)originalArray :(int)batchSize {
NSMutableArray *arrayOfArrays = [NSMutableArray array];
for(int j = 0; j < [originalArray count]; j += batchSize) {
NSArray *subarray = [originalArray subarrayWithRange:NSMakeRange(j, MIN(batchSize, [originalArray count] - j))];
[arrayOfArrays addObject:subarray];
}
return arrayOfArrays;
}
Looping through the array as follows
+(void)fetchPricelistAll:(int)pricelistId :(int)startAtRow :(int)takeNoOfRows;
{
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchPriceList:priceListId = %d", pricelistId);
}
NSString *finalURL = [NSString stringWithFormat:#"https://host.domain.com:5443/api/till/tillpricelistv2/%d?StartAtRow=%d&TakeNoOfRows=%d",pricelistId, startAtRow, takeNoOfRows];
[[[NSURLSession sharedSession] dataTaskWithURL:[NSURL URLWithString:finalURL]
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
if (error != nil) {
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchPriceList:Transport error %#", error);
}
} else {
NSHTTPURLResponse *responseHTTP;
responseHTTP = (NSHTTPURLResponse *) response;
if(responseHTTP.statusCode != 200) {
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchPriceList:Server Error %d", (int) responseHTTP.statusCode);
}
} else {
NSArray *priceListObjectArray = [NSJSONSerialization JSONObjectWithData:data
options:0
error:NULL];
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchPriceList:count = %lu", (unsigned long)[priceListObjectArray count]);
NSLog(#"WebServices:fetchPriceList:PricelistObjectArray looks like %#",priceListObjectArray);
}
AppDelegate *appDelegate = (AppDelegate *)[[UIApplication sharedApplication]delegate];
NSPersistentContainer *container = appDelegate.persistentContainer;
NSArray *arrayOfArrays = [NWTillHelper splitIntoArraysOfBatchSize:priceListObjectArray :1000];
for(NSArray *batch in arrayOfArrays) {
[container performBackgroundTask:^(NSManagedObjectContext *context ) {
context.mergePolicy = NSMergeByPropertyStoreTrumpMergePolicy;
NSDictionary *priceListObjectDict;
//Loop through the array and for each dictionary insert into local DB
for (id element in batch) {
priceListObjectDict = element;
NSString *currencyName = [priceListObjectDict objectForKey:#"currencyName"];
NSString *price = [priceListObjectDict objectForKey:#"price"];
NSString *priceIncTax = [priceListObjectDict objectForKey:#"priceIncTAX"];
NSString *validFrom = [priceListObjectDict objectForKey:#"validFromDate"];
NSString *validTo = [priceListObjectDict objectForKey:#"validToDate"];
NSString *itemId = [priceListObjectDict objectForKey:#"itemID"];
NSDateFormatter *dateFormat = [[NSDateFormatter alloc] init];
[dateFormat setDateFormat:#"YYYY-MM-dd'T'HH:mm:ss"];
NSDate *validToDate = [dateFormat dateFromString:validTo];
NSDate *validFromDate = [dateFormat dateFromString:validFrom];
if([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchPriceList:validToDate: >>>> %# <<<<", validToDate);
NSLog(#"WebServices:fetchPriceList:validFromDate: >>>> %# <<<<", validFromDate);
}
if([NWTillHelper isDebug] == 1) {
NSLog(#"PimItemListView:tableView:context = %#", context);
}
NSManagedObject *newPrlItem = Nil;
newPrlItem = [NSEntityDescription
insertNewObjectForEntityForName:#"PriceList"
inManagedObjectContext:context];
[newPrlItem setValue:itemId forKey:#"itemId"];
[newPrlItem setValue:validToDate forKey:#"validTo"];
[newPrlItem setValue:validFromDate forKey:#"validFrom"];
[newPrlItem setValue:price forKey:#"price"];
[newPrlItem setValue:priceIncTax forKey:#"priceIncTax"];
[newPrlItem setValue:currencyName forKey:#"currencyName"];
if ([NWTillHelper isDebug] == 1) {
NSLog(#"WebServices:fetchTillData:ItemId in loop = %#", itemId);
NSLog(#"WebServices:fetchTillData:newPrlItem = %#", newPrlItem);
NSLog(#"WebServices:fetchTillData:CoreData error = %#", error);
}
}
NSError *error = nil;
if (![context save:&error]) {
NSLog(#"Failure to save context: %#\n%#", [error localizedDescription], [error userInfo]);
abort();
} else {
NSUserDefaults *tillUserDefaults = [NSUserDefaults standardUserDefaults];
[tillUserDefaults setInteger:1 forKey:#"hasPriceList"];
[tillUserDefaults synchronize];
}
}];
}
}
}
}] resume];
}
It still raises to 2 GB and gets terminated, when the NSURLSession completion block hits memory usage is about 250, I assume that is after it downloads the entire data set into the NSArray, but after that when I want to write to core data all goes terribly wrong and it goes to 2 GB and gets terminated
Why is that?

Answer 1

First split up your large array into an array of arrays. Experiment with a good batch size that is not too large (that the app will crash) or too small (that will take a long time). I would suggest starting with 500. See here how do do this: What is an easy way to break an NSArray with 4000+ objects in it into multiple arrays with 30 objects each?. I assume you can turn that code into an array extension..
NSArray *arrayOfArrays = [tillBasicDataArray splitIntoArrayBatchSize:500];
Next you can enqueue many block that will each process only one of the many array and save it at the end.
for(NSArray *batch in arrayOfArrays){
[container performBackgroundTask:^(NSManagedObjectContext *context ) {
...
for (id element in batch){
...
each performBackgroundTask is enqueued into an internal operation and will process one at a time.
The rest of your code remains basically the same.