Skip to content

Commit 62aab17

Browse files
MikeMitchellWebDevMikeMitchellWebDev
committed
gc knobs
0 parents  commit 62aab17

16 files changed

+25002
-0
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gc_knobs

‎LICENSE

Lines changed: 661 additions & 0 deletions
Large diffs are not rendered by default.

‎README.md

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# gc_knobs
2+
front and back end for experimenting with Go's garbage collection tuning knobs (GOMEMLIMIT and GOGC)
3+
## How to use
4+
<p>clone the repo</p>
5+
<p>go build -o gc_knobs *.go</p>
6+
<p>./gc_knobs or GOMEMLIMIT=250MiB GOGC=50 ./gc_knobs</p>
7+
<p>open browser to localhost:8000 </p>
8+
9+
The app uses the go-git library and git repos (of your choice) that you have cloned to your computer. The app does not fetch them from github.
10+
11+
For example,
12+
to create a steady state for the garbage collector, you could read the repo for Ruby on Rails into memory 50 times with a 2 second pause in between each read.
13+
14+
curl -H 'Content-Type: application/json' -d '{"path":"/path/to/rails", "repeat":50, "pause":2}' -X POST http://localhost:8000/git_repo
15+
16+
Then, if you want to, create a transitory spike in memory, you could open another terminal window and read youtube-dl into memory once.
17+
18+
<p>curl -H 'Content-Type: application/json' -d '{"path":"/path/to/youtube-dl", "repeat":1, "pause":1}' -X POST http://localhost:8000/git_repo</p>
19+
20+
## Understanding GOMEMLIMIT and GOGC
21+
You can find tips for setting GOGC and GOMEMLIMIT at Go's garbage collection guide https://tip.golang.org/doc/gc-guide
22+
23+
## calling other functions in gc_knobs
24+
25+
curl http://localhost:8000/force_gc (calls runtime.GC)
26+
27+
curl http://localhost:8000/free_memory (calls debug.FreeOSMemory())
28+
29+
curl -H 'Content-Type: application/json' -d '{"gcpercent":75}' -X POST http://localhost:8000/set_gogc
30+
31+
curl -H 'Content-Type: application/json' -d '{"memlimit":2750, "unit":"MiB"}' -X POST http://localhost:8000/set_memlimit (available units are KiB, MiB and GiB)
32+
33+
curl http://localhost:8000/print_all_metrics (from runtime/Metrics package)
34+
35+
curl http://localhost:8000/print_some_metrics (from runtime/Metrics package)
36+
37+
the browser app uses Go's expvar package

‎all_metrics.go

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"net/http"
6+
"runtime/metrics"
7+
)
8+
9+
func printAllMetrics(w http.ResponseWriter, req *http.Request) {
10+
11+
descs := metrics.All()
12+
13+
samples := make([]metrics.Sample, len(descs))
14+
15+
for i := range samples {
16+
samples[i].Name = descs[i].Name
17+
}
18+
type Metricsresponse struct {
19+
Cpuseconds float64 `json:"/cpu/classes/gc/mark/assist:cpu-seconds"`
20+
GcCycles uint64 `json:"/gc/cycles/automatic:gc-cycles"`
21+
ForcedGcCycles uint64 `json:"/gc/cycles/forced:gc-cycles"`
22+
GOGCPercent uint64 `json:"/gc/gogc:percent"`
23+
GOMEMLIMIT uint64 `json:"/gc/gomemlimit:bytes"`
24+
HeapAllocBytes uint64 `json:"/gc/heap/allocs:bytes"`
25+
HeapAllocObjects uint64 `json:"/gc/heap/allocs:objects"`
26+
FreesBytes uint64 `json:"/gc/heap/frees:bytes"`
27+
FreesObjects uint64 `json:"/gc/heap/frees:objects"`
28+
GcHeapGoal uint64 `json:"/gc/heap/goal:bytes"`
29+
GcHeapLive uint64 `json:"/gc/heap/live:bytes"`
30+
GcHeapObjects uint64 `json:"/gc/heap/objects:objects"`
31+
GcHeapTinyObjects uint64 `json:"/gc/heap/tiny/allocs:objects"`
32+
GcLimiterEnabled uint64 `json:"/gc/limiter/last-enabled:gc-cycle"`
33+
GcScanGlobalsBytes uint64 `json:"/gc/scan/globals:bytes"`
34+
GcHeapScannableBytes uint64 `json:"/gc/scan/heap:bytes"`
35+
GcScanStackBytes uint64 `json:"/gc/scan/stack:bytes"`
36+
GcScanTotalBytes uint64 `json:"/gc/scan/total:bytes"`
37+
GcStackStartingSizeBytes uint64 `json:"/gc/stack/starting-size:bytes"`
38+
MemoryHeapFreeBytes uint64 `json:"/memory/classes/heap/free:bytes"`
39+
MemoryHeapObjectsBytes uint64 `json:"/memory/classes/heap/objects:bytes"`
40+
MemoryHeapReleasedBytes uint64 `json:"/memory/classes/heap/released:bytes"`
41+
MemoryHeapStacksBytes uint64 `json:"/memory/classes/heap/stacks:bytes"`
42+
MemoryHeapUnusedBytes uint64 `json:"/memory/classes/heap/unused:bytes"`
43+
MemoryMetaDataCacheFreeBytes uint64 `json:"/memory/classes/metadata/mcache/free:bytes"`
44+
MemoryMetaDataCacheInuseBytes uint64 `json:"/memory/classes/metadata/mcache/inuse:bytes"`
45+
MemoryMetaDataMspanFreeBytes uint64 `json:"/memory/classes/metadata/mspan/free:bytes"`
46+
MemoryMetaDataMspanInuseBytes uint64 `json:"/memory/classes/metadata/mspan/inuse:bytes"`
47+
MemoryMetaDataOtherBytes uint64 `json:"/memory/classes/metadata/other:bytes"`
48+
MemoryOsStackBytes uint64 `json:"/memory/classes/os-stacks:bytes"`
49+
MemoryOtherBytes uint64 `json:"/memory/classes/other:bytes"`
50+
MemoryProfilingBucketsBytes uint64 `json:"/memory/classes/profiling/buckets:bytes"`
51+
MemoryTotalBytes uint64 `json:"/memory/classes/total:bytes"`
52+
SchedGoMaxProcsThreads uint64 `json:"/sched/gomaxprocs:threads"`
53+
SchedGoroutines uint64 `json:"/sched/goroutines:goroutines"`
54+
SchedLatenciesSeconds float64 `json:"/sched/latencies:seconds"`
55+
SchedPausesStoppingGcSeconds float64 `json:"/sched/pauses/stopping/gc:seconds"`
56+
SchedPausesStoppingOtherSeconds float64 `json:"/sched/pauses/stopping/other:seconds"`
57+
SchedPausesTotalGcSeconds float64 `json:"/sched/pauses/total/gc:seconds"`
58+
SchedPausesTotalOtherSeconds float64 `json:"/sched/pauses/total/other:seconds"`
59+
60+
SyncMutexWaitTotalSeconds float64 `json:"/sync/mutex/wait/total:seconds"`
61+
}
62+
res := Metricsresponse{}
63+
64+
metrics.Read(samples)
65+
for _, sample := range samples {
66+
name, value := sample.Name, sample.Value
67+
switch name {
68+
case "/cpu/classes/gc/mark/assist:cpu-seconds":
69+
res.Cpuseconds = value.Float64()
70+
case "/gc/cycles/automatic:gc-cycles":
71+
res.GcCycles = value.Uint64()
72+
case "/gc/cycles/forced:gc-cycles":
73+
res.ForcedGcCycles = value.Uint64()
74+
case "/gc/gogc:percent":
75+
res.GOGCPercent = value.Uint64()
76+
case "/gc/gomemlimit:bytes":
77+
res.GOMEMLIMIT = value.Uint64()
78+
case "/gc/heap/allocs:bytes":
79+
res.HeapAllocBytes = value.Uint64()
80+
case "/gc/heap/allocs:objects":
81+
res.HeapAllocObjects = value.Uint64()
82+
case "/gc/heap/frees:bytes":
83+
res.FreesBytes = value.Uint64()
84+
case "/gc/heap/frees:objects":
85+
res.FreesObjects = value.Uint64()
86+
case "/gc/heap/goal:bytes":
87+
res.GcHeapGoal = value.Uint64()
88+
case "/gc/heap/live:bytes":
89+
res.GcHeapLive = value.Uint64()
90+
case "/gc/heap/objects:objects":
91+
res.GcHeapObjects = value.Uint64()
92+
case "/gc/heap/tiny/allocs:objects":
93+
res.GcHeapTinyObjects = value.Uint64()
94+
case "/gc/limiter/last-enabled:gc-cycle":
95+
res.GcLimiterEnabled = value.Uint64()
96+
case "/gc/scan/globals:bytes":
97+
res.GcScanGlobalsBytes = value.Uint64()
98+
case "/gc/scan/heap:bytes":
99+
res.GcHeapScannableBytes = value.Uint64()
100+
case "/gc/scan/stack:bytes":
101+
res.GcScanStackBytes = value.Uint64()
102+
case "/gc/scan/total:bytes":
103+
res.GcScanTotalBytes = value.Uint64()
104+
case "/gc/stack/starting-size:bytes":
105+
res.GcStackStartingSizeBytes = value.Uint64()
106+
case "/memory/classes/heap/free:bytes":
107+
res.MemoryHeapFreeBytes = value.Uint64()
108+
case "/memory/classes/heap/objects:bytes":
109+
res.MemoryHeapObjectsBytes = value.Uint64()
110+
case "/memory/classes/heap/released:bytes":
111+
res.MemoryHeapReleasedBytes = value.Uint64()
112+
case "/memory/classes/heap/stacks:bytes":
113+
res.MemoryHeapStacksBytes = value.Uint64()
114+
case "/memory/classes/heap/unused:bytes":
115+
res.MemoryHeapUnusedBytes = value.Uint64()
116+
case "/memory/classes/metadata/mcache/free:bytes":
117+
res.MemoryMetaDataCacheFreeBytes = value.Uint64()
118+
case "/memory/classes/metadata/mcache/inuse:bytes":
119+
res.MemoryMetaDataCacheInuseBytes = value.Uint64()
120+
case "/memory/classes/metadata/mspan/free:bytes":
121+
res.MemoryMetaDataMspanFreeBytes = value.Uint64()
122+
case "/memory/classes/metadata/mspan/inuse:bytes":
123+
res.MemoryMetaDataMspanInuseBytes = value.Uint64()
124+
case "/memory/classes/metadata/other:bytes":
125+
res.MemoryMetaDataOtherBytes = value.Uint64()
126+
case "/memory/classes/os-stacks:bytes":
127+
res.MemoryOsStackBytes = value.Uint64()
128+
case "/memory/classes/other:bytes":
129+
res.MemoryOtherBytes = value.Uint64()
130+
case "/memory/classes/profiling/buckets:bytes":
131+
res.MemoryProfilingBucketsBytes = value.Uint64()
132+
case "/memory/classes/total:bytes":
133+
res.MemoryTotalBytes = value.Uint64()
134+
case "/sched/gomaxprocs:threads":
135+
res.SchedGoMaxProcsThreads = value.Uint64()
136+
case "/sched/goroutines:goroutines":
137+
res.SchedGoroutines = value.Uint64()
138+
case "/sched/latencies:seconds":
139+
res.SchedLatenciesSeconds = medianBucket(value.Float64Histogram())
140+
case "/sched/pauses/stopping/gc:seconds":
141+
res.SchedPausesStoppingGcSeconds = medianBucket(value.Float64Histogram())
142+
case "/sched/pauses/stopping/other:seconds":
143+
res.SchedPausesStoppingOtherSeconds = medianBucket(value.Float64Histogram())
144+
case "/sched/pauses/total/gc:seconds":
145+
res.SchedPausesTotalGcSeconds = medianBucket(value.Float64Histogram())
146+
case "/sched/pauses/total/other:seconds":
147+
res.SchedPausesTotalOtherSeconds = medianBucket(value.Float64Histogram())
148+
case "/sync/mutex/wait/total:seconds":
149+
res.SyncMutexWaitTotalSeconds = value.Float64()
150+
default:
151+
}
152+
}
153+
154+
samples = nil
155+
156+
json.NewEncoder(w).Encode(res)
157+
158+
}
159+
160+
func medianBucket(h *metrics.Float64Histogram) float64 {
161+
total := uint64(0)
162+
for _, count := range h.Counts {
163+
total += count
164+
}
165+
thresh := total / 2
166+
total = 0
167+
for i, count := range h.Counts {
168+
total += count
169+
if total >= thresh {
170+
return h.Buckets[i]
171+
}
172+
}
173+
panic("should not happen")
174+
}

0 commit comments

Comments
 (0)