How to extract performance metrics measured by measureBlock in XCTest - objective-c

Question

I have a simple test function which will tap a button an measure the performance. I'm using XCTest. After measureBlock returns I can see a bunch of perf-metrics on the console. I would like to get this within the test-program such that I can populate the data somewhere else programmatically. Watching the test data on test-console is proving to be slow because I have a lot of test-cases.
- (void)testUseMeasureBlock {
XCUIElement *launchTest1Button = [[XCUIApplication alloc] init].buttons[#"Launch Test 1"];
void (^blockToMeasure)(void) = ^void(void) {
[launchTest1Button tap];
};
// Run once to warm up any potential caching properties
#autoreleasepool {
blockToMeasure();
}
// Now measure the block
[self measureBlock:blockToMeasure];
/// Collect the measured metrics and send somewhere.
When we run a test it prints:
measured [Time, seconds] average: 0.594, relative standard deviation: 0.517%, values: [0.602709, 0.593631, 0.593004, 0.592350, 0.596199, 0.593807, 0.591444, 0.593460, 0.592648, 0.592769],
If I could get the average time, that'd be sufficient for now.

Answer 1

Since there is no API to get this data you can pipe stderr stream and parse tests logs to get needed info e.g. average time. For instance you can use next approach:
#interface MeasureParser : NSObject
#property (nonatomic) NSPipe* pipe;
#property (nonatomic) NSRegularExpression* regex;
#property (nonatomic) NSMutableDictionary* results;
#end
#implementation MeasureParser
- (instancetype)init {
self = [super self];
if (self) {
self.pipe = NSPipe.pipe;
self.results = [NSMutableDictionary new];
let pattern = [NSString stringWithFormat:#"[^']+'\\S+\\s([^\\]]+)\\]'\\smeasured\\s\\[Time,\\sseconds\\]\\saverage:\\s([^,]+)"];
NSError* error = nil;
self.regex = [NSRegularExpression regularExpressionWithPattern:pattern options:NSRegularExpressionCaseInsensitive error:&error];
if (error) {
return nil;
}
}
return self;
}
- (void)capture:(void (^)(void))block {
// Save original output
int original = dup(STDERR_FILENO);
setvbuf(stderr, nil, _IONBF, 0);
dup2(self.pipe.fileHandleForWriting.fileDescriptor, STDERR_FILENO);
__weak let wself = self;
self.pipe.fileHandleForReading.readabilityHandler = ^(NSFileHandle *handle) {
var *str = [[NSString alloc] initWithData:handle.availableData encoding:NSUTF8StringEncoding];
let firstMatch = [wself.regex firstMatchInString:str options:NSMatchingReportCompletion range:NSMakeRange(0, str.length)];
if (firstMatch) {
let name = [str substringWithRange:[firstMatch rangeAtIndex:1]];
let average = [str substringWithRange:[firstMatch rangeAtIndex:2]];
wself.results[name] = average;
}
// Print to stdout because stderr is piped
printf("%s", [str cStringUsingEncoding:NSUTF8StringEncoding]);
};
block();
// Revert
fflush(stderr);
dup2(original, STDERR_FILENO);
close(original);
}
#end
How to use:
- (void)testPerformanceExample {
let measureParser = [MeasureParser new];
[measureParser capture:^{
[self measureBlock:^{
// Put the code you want to measure the time of here.
sleep(1);
}];
}];
NSLog(#"%#", measureParser.results);
}
// Outputs
{
testPerformanceExample = "1.001";
}

Answer 2

Swift 5 version
final class MeasureParser {
let pipe: Pipe = Pipe()
let regex: NSRegularExpression?
let results: NSMutableDictionary = NSMutableDictionary()
init() {
self.regex = try? NSRegularExpression(
pattern: "\\[(Clock Monotonic Time|CPU Time|Memory Peak Physical|Memory Physical|CPU Instructions Retired|Disk Logical Writes|CPU Cycles), (s|kB|kI|kC)\\] average: ([0-9\\.]*),",
options: .caseInsensitive)
}
func capture(completion: #escaping () -> Void) {
let original = dup(STDERR_FILENO)
setvbuf(stderr, nil, _IONBF, 0)
dup2(self.pipe.fileHandleForWriting.fileDescriptor, STDERR_FILENO)
self.pipe.fileHandleForReading.readabilityHandler = { [weak self] handle in
guard self != nil else { return }
let data = handle.availableData
let str = String(data: data, encoding: .utf8) ?? "<Non-ascii data of size\(data.count)>\n"
self!.fetchAndSaveMetrics(str)
// Print to stdout because stderr is piped
if let copy = (str as NSString?)?.cString(using: String.Encoding.utf8.rawValue) {
print("\(copy)")
}
}
completion()
fflush(stderr)
dup2(original, STDERR_FILENO)
close(original)
}
private func fetchAndSaveMetrics(_ str: String) {
guard let mRegex = self.regex else { return }
let matches = mRegex.matches(in: str, options: .reportCompletion, range: NSRange(location: 0, length: str.count))
matches.forEach {
let nameIndex = Range($0.range(at: 1), in: str)
let averageIndex = Range($0.range(at: 3), in: str)
if nameIndex != nil && averageIndex != nil {
let name = str[nameIndex!]
let average = str[averageIndex!]
self.results[name] = average
}
}
}
}
How to use it:
import XCTest
final class MyUiTests: XCTestCase {
var app: XCUIApplication!
let measureParser = MeasureParser()
// MARK: - XCTestCase
override func setUp() {
super.setUp()
continueAfterFailure = false
app = XCUIApplication()
app.launch()
}
override func tearDown() {
//FIXME: Just for debugging
print(self.measureParser.results)
print(self.measureParser.results["CPU Cycles"])
print(self.measureParser.results["CPU Instructions Retired"])
print(self.measureParser.results["CPU Time"])
print(self.measureParser.results["Clock Monotonic Time"])
print(self.measureParser.results["Disk Logical Writes"])
print(self.measureParser.results["Memory Peak Physical"])
print(self.measureParser.results["Memory Physical"])
}
// MARK: - Tests
func testListing() {
self.measureParser.capture { [weak self] in
guard let self = self else { return }
self.measureListingScroll()
}
}
// MARK: XCTest measures
private func measureListingScroll() {
measure(metrics: [XCTCPUMetric(), XCTClockMetric(), XCTMemoryMetric(), XCTStorageMetric()]) {
self.app.swipeUp()
self.app.swipeUp()
self.app.swipeUp()
}
}
}

Answer 3

There's a private instance variable __perfMetricsForID of XCTestCase store the result.
And you can access it by call
NSDictionary* perfMetrics = [testCase valueForKey:#"__perfMetricsForID"];
the result is just like this: