Skip to content

Commit 7d89675

Browse files
dd-mergequeue[bot]maxep
dd-mergequeue[bot]
and
maxep
authored
Merge pull request #2218 from DataDog/maxep/RUM-8442/add-benchmark-meters
RUM-8442 Add Benchmark Metric Meters Co-authored-by: maxep <[email protected]>
2 parents c8d899f + d428848 commit 7d89675

File tree

8 files changed

+266
-102
lines changed

8 files changed

+266
-102
lines changed

‎BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
D231DCF52C73356E00F3F66C /* WebViewController.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = D231DCAA2C73356D00F3F66C /* WebViewController.storyboard */; };
8383
D231DCF62C73356E00F3F66C /* WebViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D231DCAB2C73356D00F3F66C /* WebViewController.swift */; };
8484
D231DCF92C7342D500F3F66C /* ModuleBundle.swift in Sources */ = {isa = PBXBuildFile; fileRef = D231DCF82C7342D500F3F66C /* ModuleBundle.swift */; };
85+
D23734972D773DD8004CCAED /* BenchmarkMeter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D23734962D773DD1004CCAED /* BenchmarkMeter.swift */; };
8586
D23DD32D2C58D80C00B90C4C /* DatadogBenchmarks in Frameworks */ = {isa = PBXBuildFile; productRef = D23DD32C2C58D80C00B90C4C /* DatadogBenchmarks */; };
8687
D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */ = {isa = PBXBuildFile; fileRef = D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */; };
8788
D24E15F32C776956005AE4E8 /* BenchmarkProfiler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D24E15F22C776956005AE4E8 /* BenchmarkProfiler.swift */; };
@@ -312,6 +313,7 @@
312313
D231DCAB2C73356D00F3F66C /* WebViewController.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = WebViewController.swift; sourceTree = "<group>"; };
313314
D231DCF82C7342D500F3F66C /* ModuleBundle.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModuleBundle.swift; sourceTree = "<group>"; };
314315
D231DCFA2C735FC200F3F66C /* LICENSE.txt */ = {isa = PBXFileReference; lastKnownFileType = text; path = LICENSE.txt; sourceTree = "<group>"; };
316+
D23734962D773DD1004CCAED /* BenchmarkMeter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BenchmarkMeter.swift; sourceTree = "<group>"; };
315317
D24BFD462C6B916B00AB9604 /* SyntheticScenario.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyntheticScenario.swift; sourceTree = "<group>"; };
316318
D24E15F22C776956005AE4E8 /* BenchmarkProfiler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BenchmarkProfiler.swift; sourceTree = "<group>"; };
317319
D26DD6E62D0C774000D96148 /* DatadogMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DatadogMonitor.swift; sourceTree = "<group>"; };
@@ -922,6 +924,7 @@
922924
D29F754F2C4AA07E00288638 /* AppDelegate.swift */,
923925
D276069D2C514F37002D2A14 /* AppConfiguration.swift */,
924926
D24E15F22C776956005AE4E8 /* BenchmarkProfiler.swift */,
927+
D23734962D773DD1004CCAED /* BenchmarkMeter.swift */,
925928
D276069C2C514F37002D2A14 /* Scenarios */,
926929
D29F755D2C4AA08000288638 /* Info.plist */,
927930
);
@@ -1285,6 +1288,7 @@
12851288
D24E15F32C776956005AE4E8 /* BenchmarkProfiler.swift in Sources */,
12861289
D24BFD472C6B916B00AB9604 /* SyntheticScenario.swift in Sources */,
12871290
D27606A32C514F37002D2A14 /* Scenario.swift in Sources */,
1291+
D23734972D773DD8004CCAED /* BenchmarkMeter.swift in Sources */,
12881292
);
12891293
runOnlyForDeploymentPostprocessing = 0;
12901294
};

‎BenchmarkTests/Benchmarks/Sources/Benchmarks.swift

Lines changed: 17 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ import OpenTelemetryApi
1313
import OpenTelemetrySdk
1414
import DatadogExporter
1515

16-
let instrumentationName = "benchmarks"
17-
let instrumentationVersion = "1.0.0"
18-
1916
/// Benchmark entrypoint to configure opentelemetry with metrics meters
2017
/// and tracer.
2118
public enum Benchmarks {
@@ -75,79 +72,38 @@ public enum Benchmarks {
7572
}
7673
}
7774

78-
/// Configure OpenTelemetry metrics meter and start measuring Memory.
75+
/// Configure an OpenTelemetry meter provider.
7976
///
8077
/// - Parameter configuration: The Benchmark configuration.
81-
public static func enableMetrics(with configuration: Configuration) {
78+
public static func meterProvider(with configuration: Configuration) -> MeterProvider {
8279
let metricExporter = MetricExporter(
8380
configuration: MetricExporter.Configuration(
8481
apiKey: configuration.apiKey,
85-
version: instrumentationVersion
82+
version: configuration.context.applicationVersion
8683
)
8784
)
8885

89-
let meterProvider = MeterProviderBuilder()
86+
return MeterProviderBuilder()
9087
.with(pushInterval: 10)
9188
.with(processor: MetricProcessorSdk())
9289
.with(exporter: metricExporter)
93-
.with(resource: Resource())
90+
.with(resource: Resource(attributes: [
91+
"device_model": .string(configuration.context.deviceModel),
92+
"os": .string(configuration.context.osName),
93+
"os_version": .string(configuration.context.osVersion),
94+
"run": .string(configuration.context.run),
95+
"scenario": .string(configuration.context.scenario),
96+
"application_id": .string(configuration.context.applicationIdentifier),
97+
"sdk_version": .string(configuration.context.sdkVersion),
98+
"branch": .string(configuration.context.branch),
99+
]))
94100
.build()
95-
96-
let meter = meterProvider.get(
97-
instrumentationName: instrumentationName,
98-
instrumentationVersion: instrumentationVersion
99-
)
100-
101-
let labels = [
102-
"device_model": configuration.context.deviceModel,
103-
"os": configuration.context.osName,
104-
"os_version": configuration.context.osVersion,
105-
"run": configuration.context.run,
106-
"scenario": configuration.context.scenario,
107-
"application_id": configuration.context.applicationIdentifier,
108-
"sdk_version": configuration.context.sdkVersion,
109-
"branch": configuration.context.branch,
110-
]
111-
112-
let queue = DispatchQueue(label: "com.datadoghq.benchmarks.metrics", qos: .utility)
113-
114-
let memory = Memory(queue: queue)
115-
_ = meter.createDoubleObservableGauge(name: "ios.benchmark.memory") { metric in
116-
// report the maximum memory footprint that was recorded during push interval
117-
if let value = memory.aggregation?.max {
118-
metric.observe(value: value, labels: labels)
119-
}
120-
121-
memory.reset()
122-
}
123-
124-
let cpu = CPU(queue: queue)
125-
_ = meter.createDoubleObservableGauge(name: "ios.benchmark.cpu") { metric in
126-
// report the average cpu usage that was recorded during push interval
127-
if let value = cpu.aggregation?.avg {
128-
metric.observe(value: value, labels: labels)
129-
}
130-
131-
cpu.reset()
132-
}
133-
134-
let fps = FPS()
135-
_ = meter.createIntObservableGauge(name: "ios.benchmark.fps.min") { metric in
136-
// report the minimum frame rate that was recorded during push interval
137-
if let value = fps.aggregation?.min {
138-
metric.observe(value: value, labels: labels)
139-
}
140-
141-
fps.reset()
142-
}
143-
144-
OpenTelemetry.registerMeterProvider(meterProvider: meterProvider)
145101
}
146102

147-
/// Configure and register a OpenTelemetry Tracer.
103+
/// Configure an OpenTelemetry tracer provider.
148104
///
149105
/// - Parameter configuration: The Benchmark configuration.
150-
public static func enableTracer(with configuration: Configuration) {
106+
public static func tracerProvider(with configuration: Configuration) -> TracerProvider {
151107
let exporterConfiguration = ExporterConfiguration(
152108
serviceName: configuration.context.applicationIdentifier,
153109
resource: "Benchmark Tracer",
@@ -162,10 +118,8 @@ public enum Benchmarks {
162118
let exporter = try! DatadogExporter(config: exporterConfiguration)
163119
let processor = SimpleSpanProcessor(spanExporter: exporter)
164120

165-
let provider = TracerProviderBuilder()
121+
return TracerProviderBuilder()
166122
.add(spanProcessor: processor)
167123
.build()
168-
169-
OpenTelemetry.registerTracerProvider(tracerProvider: provider)
170124
}
171125
}

‎BenchmarkTests/Benchmarks/Sources/Metrics.swift

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,33 +12,33 @@ import QuartzCore
1212
let TASK_VM_INFO_COUNT = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)
1313
let TASK_VM_INFO_REV1_COUNT = mach_msg_type_number_t(MemoryLayout.offset(of: \task_vm_info_data_t.min_address)! / MemoryLayout<integer_t>.size)
1414

15-
internal enum MachError: Error {
15+
public enum MachError: Error {
1616
case task_info(return: kern_return_t)
1717
case task_threads(return: kern_return_t)
1818
case thread_info(return: kern_return_t)
1919
}
2020

2121
/// Aggregate metric values and compute `min`, `max`, `sum`, `avg`, and `count`.
22-
internal class MetricAggregator<T> where T: Numeric {
23-
internal struct Aggregation {
24-
let min: T
25-
let max: T
26-
let sum: T
27-
let count: Int
28-
let avg: Double
22+
public class MetricAggregator<T> where T: Numeric {
23+
public struct Aggregation {
24+
public let min: T
25+
public let max: T
26+
public let sum: T
27+
public let count: Int
28+
public let avg: Double
2929
}
3030

3131
private var mutex = pthread_mutex_t()
3232
private var _aggregation: Aggregation?
3333

34-
var aggregation: Aggregation? {
34+
public var aggregation: Aggregation? {
3535
pthread_mutex_lock(&mutex)
3636
defer { pthread_mutex_unlock(&mutex) }
3737
return _aggregation
3838
}
3939

4040
/// Resets the minimum frame rate to `nil`.
41-
func reset() {
41+
public func reset() {
4242
pthread_mutex_lock(&mutex)
4343
_aggregation = nil
4444
pthread_mutex_unlock(&mutex)
@@ -53,7 +53,7 @@ extension MetricAggregator where T: BinaryInteger {
5353
/// Records a `BinaryInteger` value.
5454
///
5555
/// - Parameter value: The value to record.
56-
func record(value: T) {
56+
public func record(value: T) {
5757
pthread_mutex_lock(&mutex)
5858
_aggregation = _aggregation.map {
5959
let sum = $0.sum + value
@@ -74,7 +74,7 @@ extension MetricAggregator where T: BinaryFloatingPoint {
7474
/// Records a `BinaryFloatingPoint` value.
7575
///
7676
/// - Parameter value: The value to record.
77-
func record(value: T) {
77+
internal func record(value: T) {
7878
pthread_mutex_lock(&mutex)
7979
_aggregation = _aggregation.map {
8080
let sum = $0.sum + value
@@ -94,7 +94,7 @@ extension MetricAggregator where T: BinaryFloatingPoint {
9494
/// Collect Memory footprint metric.
9595
///
9696
/// Based on a timer, the `Memory` aggregator will periodically record the memory footprint.
97-
internal final class Memory: MetricAggregator<Double> {
97+
public final class Memory: MetricAggregator<Double> {
9898
/// Dispatch source object for monitoring timer events.
9999
private let timer: DispatchSourceTimer
100100

@@ -107,7 +107,7 @@ internal final class Memory: MetricAggregator<Double> {
107107
/// - queue: The queue on which to execute the timer handler.
108108
/// - interval: The timer interval, default to 100 ms.
109109
/// - leeway: The timer leeway, default to 10 ms.
110-
required init(
110+
public required init(
111111
queue: DispatchQueue,
112112
every interval: DispatchTimeInterval = .milliseconds(100),
113113
leeway: DispatchTimeInterval = .milliseconds(10)
@@ -158,7 +158,7 @@ internal final class Memory: MetricAggregator<Double> {
158158
/// Collect CPU usage metric.
159159
///
160160
/// Based on a timer, the `CPU` aggregator will periodically record the CPU usage.
161-
internal final class CPU: MetricAggregator<Double> {
161+
public final class CPU: MetricAggregator<Double> {
162162
/// Dispatch source object for monitoring timer events.
163163
private let timer: DispatchSourceTimer
164164

@@ -171,7 +171,7 @@ internal final class CPU: MetricAggregator<Double> {
171171
/// - queue: The queue on which to execute the timer handler.
172172
/// - interval: The timer interval, default to 100 ms.
173173
/// - leeway: The timer leeway, default to 10 ms.
174-
init(
174+
public required init(
175175
queue: DispatchQueue,
176176
every interval: DispatchTimeInterval = .milliseconds(100),
177177
leeway: DispatchTimeInterval = .milliseconds(10)
@@ -241,7 +241,7 @@ internal final class CPU: MetricAggregator<Double> {
241241
}
242242

243243
/// Collect Frame rate metric based on ``CADisplayLinker`` timer.
244-
internal final class FPS: MetricAggregator<Int> {
244+
public final class FPS: MetricAggregator<Int> {
245245
private class CADisplayLinker {
246246
weak var fps: FPS?
247247

@@ -260,7 +260,7 @@ internal final class FPS: MetricAggregator<Int> {
260260

261261
private var displayLink: CADisplayLink
262262

263-
override init() {
263+
override public init() {
264264
let linker = CADisplayLinker()
265265
displayLink = CADisplayLink(target: linker, selector: #selector(CADisplayLinker.tick(link:)))
266266
super.init()

‎BenchmarkTests/Runner/AppDelegate.swift

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,36 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
2020
let run = SyntheticRun()
2121
let applicationInfo = try! AppInfo() // crash if info are missing or malformed
2222

23-
switch run {
24-
case .baseline, .instrumented:
25-
// measure metrics during baseline and metrics runs
26-
Benchmarks.enableMetrics(
23+
// Collect metrics during all run
24+
let meter = Meter(
25+
provider: Benchmarks.meterProvider(
2726
with: Benchmarks.Configuration(
2827
info: applicationInfo,
2928
scenario: scenario,
3029
run: run
3130
)
3231
)
32+
)
33+
34+
switch run {
35+
case .baseline, .instrumented:
36+
meter.observeCPU()
37+
meter.observeMemory()
38+
meter.observeFPS()
39+
3340
case .profiling:
3441
// Collect traces during profiling run
35-
Benchmarks.enableTracer(
36-
with: Benchmarks.Configuration(
37-
info: applicationInfo,
38-
scenario: scenario,
39-
run: run
42+
let profiler = Profiler(
43+
provider: Benchmarks.tracerProvider(
44+
with: Benchmarks.Configuration(
45+
info: applicationInfo,
46+
scenario: scenario,
47+
run: run
48+
)
4049
)
4150
)
4251

43-
DatadogInternal.profiler = Profiler()
52+
DatadogInternal.bench = (profiler, meter)
4453
case .none:
4554
break
4655
}

‎BenchmarkTests/Runner/BenchmarkMeter.swift

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2019-Present Datadog, Inc.
5+
*/
6+
7+
import Foundation
8+
import DatadogInternal
9+
import DatadogBenchmarks
10+
import OpenTelemetryApi
11+
import OpenTelemetrySdk
12+
13+
internal final class Meter: DatadogInternal.BenchmarkMeter {
14+
let meter: OpenTelemetryApi.Meter
15+
16+
let queue = DispatchQueue(label: "com.datadoghq.benchmarks.metrics", target: .global(qos: .utility))
17+
18+
init(provider: MeterProvider) {
19+
self.meter = provider.get(
20+
instrumentationName: "benchmarks",
21+
instrumentationVersion: nil
22+
)
23+
}
24+
25+
@discardableResult
26+
func observeMemory() -> OpenTelemetryApi.DoubleObserverMetric {
27+
let memory = Memory(queue: queue)
28+
return meter.createDoubleObservableGauge(name: "ios.benchmark.memory") { metric in
29+
// report the maximum memory footprint that was recorded during push interval
30+
if let value = memory.aggregation?.max {
31+
metric.observe(value: value, labelset: .empty)
32+
}
33+
34+
memory.reset()
35+
}
36+
}
37+
38+
@discardableResult
39+
func observeCPU() -> OpenTelemetryApi.DoubleObserverMetric {
40+
let cpu = CPU(queue: queue)
41+
return meter.createDoubleObservableGauge(name: "ios.benchmark.cpu") { metric in
42+
// report the average cpu usage that was recorded during push interval
43+
if let value = cpu.aggregation?.avg {
44+
metric.observe(value: value, labelset: .empty)
45+
}
46+
47+
cpu.reset()
48+
}
49+
}
50+
51+
@discardableResult
52+
func observeFPS() -> OpenTelemetryApi.IntObserverMetric {
53+
let fps = FPS()
54+
return meter.createIntObservableGauge(name: "ios.benchmark.fps.min") { metric in
55+
// report the minimum frame rate that was recorded during push interval
56+
if let value = fps.aggregation?.min {
57+
metric.observe(value: value, labelset: .empty)
58+
}
59+
60+
fps.reset()
61+
}
62+
}
63+
64+
func counter(metric: @autoclosure () -> String) -> DatadogInternal.BenchmarkCounter {
65+
meter.createDoubleCounter(name: metric())
66+
}
67+
68+
func gauge(metric: @autoclosure () -> String) -> DatadogInternal.BenchmarkGauge {
69+
meter.createDoubleMeasure(name: metric())
70+
}
71+
}
72+
73+
extension AnyCounterMetric<Double>: DatadogInternal.BenchmarkCounter {
74+
public func add(value: Double, attributes: @autoclosure () -> [String: String]) {
75+
add(value: value, labelset: LabelSet(labels: attributes()))
76+
}
77+
}
78+
79+
extension AnyMeasureMetric<Double>: DatadogInternal.BenchmarkGauge {
80+
public func record(value: Double, attributes: @autoclosure () -> [String: String]) {
81+
record(value: value, labelset: LabelSet(labels: attributes()))
82+
}
83+
}

0 commit comments

Comments
 (0)