Skip to content

Commit 3b9cce4

Browse files
authored
feat: improved resource manager based on Rainbow (#67)
1 parent a9caa78 commit 3b9cce4

File tree

10 files changed

+313
-35
lines changed

10 files changed

+313
-35
lines changed

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ The following emojis are used to highlight certain changes:
1717

1818
### Changed
1919

20+
- The resource manager's defaults have been improved based on Rainbow's and Kubo's defaults. In addition, you can now customize a few options using flags, or [environment variables](./docs/environment-variables.md).
21+
2022
### Removed
2123

2224
### Fixed

docs/environment-variables.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
- [`SOMEGUY_PROVIDER_ENDPOINTS`](#someguy_provider_endpoints)
99
- [`SOMEGUY_PEER_ENDPOINTS`](#someguy_peer_endpoints)
1010
- [`SOMEGUY_IPNS_ENDPOINTS`](#someguy_ipns_endpoints)
11+
- [`SOMEGUY_LIBP2P_CONNMGR_LOW`](#someguy_libp2p_connmgr_low)
12+
- [`SOMEGUY_LIBP2P_CONNMGR_HIGH`](#someguy_libp2p_connmgr_high)
13+
- [`SOMEGUY_LIBP2P_CONNMGR_GRACE_PERIOD`](#someguy_libp2p_connmgr_grace_period)
14+
- [`SOMEGUY_LIBP2P_MAX_MEMORY`](#someguy_libp2p_max_memory)
15+
- [`SOMEGUY_LIBP2P_MAX_FD`](#someguy_libp2p_max_fd)
1116
- [Logging](#logging)
1217
- [`GOLOG_LOG_LEVEL`](#golog_log_level)
1318
- [`GOLOG_LOG_FMT`](#golog_log_fmt)
@@ -46,6 +51,36 @@ Comma-separated list of other Delegated Routing V1 endpoints to proxy IPNS reque
4651

4752
Default: none
4853

54+
### `SOMEGUY_LIBP2P_CONNMGR_LOW`
55+
56+
Minimum number of libp2p connections to keep.
57+
58+
Default: 100
59+
60+
### `SOMEGUY_LIBP2P_CONNMGR_HIGH`
61+
62+
Maximum number of libp2p connections to keep.
63+
64+
Default: 3000
65+
66+
### `SOMEGUY_LIBP2P_CONNMGR_GRACE_PERIOD`
67+
68+
Minimum libp2p connection TTL.
69+
70+
Default: 1m
71+
72+
### `SOMEGUY_LIBP2P_MAX_MEMORY`
73+
74+
Maximum memory to use for libp2p.
75+
76+
Default: 0 (85% of the system's available RAM)
77+
78+
### `SOMEGUY_LIBP2P_MAX_FD`
79+
80+
Maximum number of file descriptors used by libp2p node.
81+
82+
Default: 0 (50% of the process' limit)
83+
4984
## Logging
5085

5186
### `GOLOG_LOG_LEVEL`

go.mod

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.21
44

55
require (
66
github.com/CAFxX/httpcompression v0.0.9
7+
github.com/dustin/go-humanize v1.0.1
78
github.com/felixge/httpsnoop v1.0.4
89
github.com/ipfs/boxo v0.19.1-0.20240515083429-ac0bab3926a8
910
github.com/ipfs/go-cid v0.4.1
@@ -14,11 +15,13 @@ require (
1415
github.com/multiformats/go-multiaddr v0.12.3
1516
github.com/multiformats/go-multibase v0.2.0
1617
github.com/multiformats/go-multihash v0.2.3
18+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
1719
github.com/prometheus/client_golang v1.19.0
1820
github.com/rs/cors v1.10.1
1921
github.com/slok/go-http-metrics v0.11.0
2022
github.com/stretchr/testify v1.9.0
2123
github.com/urfave/cli/v2 v2.27.1
24+
golang.org/x/sys v0.19.0
2225
)
2326

2427
require (
@@ -89,7 +92,6 @@ require (
8992
github.com/onsi/ginkgo/v2 v2.17.1 // indirect
9093
github.com/opencontainers/runtime-spec v1.2.0 // indirect
9194
github.com/opentracing/opentracing-go v1.2.0 // indirect
92-
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
9395
github.com/pkg/errors v0.9.1 // indirect
9496
github.com/pmezard/go-difflib v1.0.0 // indirect
9597
github.com/polydawn/refmt v0.89.0 // indirect
@@ -120,7 +122,6 @@ require (
120122
golang.org/x/mod v0.17.0 // indirect
121123
golang.org/x/net v0.24.0 // indirect
122124
golang.org/x/sync v0.7.0 // indirect
123-
golang.org/x/sys v0.19.0 // indirect
124125
golang.org/x/text v0.14.0 // indirect
125126
golang.org/x/tools v0.20.0 // indirect
126127
gonum.org/v1/gonum v0.15.0 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
7575
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
7676
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
7777
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
78+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
79+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
7880
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
7981
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
8082
github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=

internal/fd/sys_not_unix.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
//go:build !linux && !darwin && !windows
2+
3+
package fd
4+
5+
func GetNumFDs() int {
6+
return 0
7+
}

internal/fd/sys_unix.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
//go:build linux || darwin
2+
// +build linux darwin
3+
4+
// Package fd provides filesystem descriptor count for different architectures.
5+
package fd
6+
7+
import (
8+
"golang.org/x/sys/unix"
9+
)
10+
11+
func GetNumFDs() int {
12+
var l unix.Rlimit
13+
if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &l); err != nil {
14+
return 0
15+
}
16+
return int(l.Cur)
17+
}

internal/fd/sys_windows.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
//go:build windows
2+
3+
package fd
4+
5+
import (
6+
"math"
7+
)
8+
9+
func GetNumFDs() int {
10+
return math.MaxInt
11+
}

main.go

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"errors"
55
"log"
66
"os"
7+
"time"
78

89
"github.com/ipfs/boxo/ipns"
910
"github.com/ipfs/go-cid"
@@ -53,9 +54,54 @@ func main() {
5354
EnvVars: []string{"SOMEGUY_IPNS_ENDPOINTS"},
5455
Usage: "other Delegated Routing V1 endpoints to proxy IPNS requests to",
5556
},
57+
&cli.IntFlag{
58+
Name: "libp2p-connmgr-low",
59+
Value: 100,
60+
EnvVars: []string{"SOMEGUY_LIBP2P_CONNMGR_LOW"},
61+
Usage: "minimum number of libp2p connections to keep",
62+
},
63+
&cli.IntFlag{
64+
Name: "libp2p-connmgr-high",
65+
Value: 3000,
66+
EnvVars: []string{"SOMEGUY_LIBP2P_CONNMGR_HIGH"},
67+
Usage: "maximum number of libp2p connections to keep",
68+
},
69+
&cli.DurationFlag{
70+
Name: "libp2p-connmgr-grace",
71+
Value: time.Minute,
72+
EnvVars: []string{"SOMEGUY_LIBP2P_CONNMGR_GRACE_PERIOD"},
73+
Usage: "minimum libp2p connection TTL",
74+
},
75+
&cli.Uint64Flag{
76+
Name: "libp2p-max-memory",
77+
Value: 0,
78+
EnvVars: []string{"SOMEGUY_LIBP2P_MAX_MEMORY"},
79+
Usage: "maximum memory to use for libp2p. Defaults to 85% of the system's available RAM",
80+
},
81+
&cli.Uint64Flag{
82+
Name: "libp2p-max-fd",
83+
Value: 0,
84+
EnvVars: []string{"SOMEGUY_LIBP2P_MAX_FD"},
85+
Usage: "maximum number of file descriptors used by libp2p node. Defaults to 50% of the process' limit",
86+
},
5687
},
5788
Action: func(ctx *cli.Context) error {
58-
return start(ctx.Context, ctx.String("listen-address"), ctx.Bool("accelerated-dht"), ctx.StringSlice("provider-endpoints"), ctx.StringSlice("peer-endpoints"), ctx.StringSlice("ipns-endpoints"))
89+
cfg := &config{
90+
listenAddress: ctx.String("listen-address"),
91+
acceleratedDHTClient: ctx.Bool("accelerated-dht"),
92+
93+
contentEndpoints: ctx.StringSlice("provider-endpoints"),
94+
peerEndpoints: ctx.StringSlice("peer-endpoints"),
95+
ipnsEndpoints: ctx.StringSlice("ipns-endpoints"),
96+
97+
connMgrLow: ctx.Int("libp2p-connmgr-low"),
98+
connMgrHi: ctx.Int("libp2p-connmgr-high"),
99+
connMgrGrace: ctx.Duration("libp2p-connmgr-grace"),
100+
maxMemory: ctx.Uint64("libp2p-max-memory"),
101+
maxFD: ctx.Int("libp2p-max-fd"),
102+
}
103+
104+
return start(ctx.Context, cfg)
59105
},
60106
},
61107
{

rcmgr.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package main
2+
3+
import (
4+
"github.com/dustin/go-humanize"
5+
"github.com/pbnjay/memory"
6+
7+
"github.com/ipfs/someguy/internal/fd"
8+
"github.com/libp2p/go-libp2p"
9+
"github.com/libp2p/go-libp2p/core/network"
10+
rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
11+
)
12+
13+
// Note: this comes from rainbow/rcmgr.go with minimal adaptations.
14+
15+
var infiniteResourceLimits = rcmgr.InfiniteLimits.ToPartialLimitConfig().System
16+
17+
func makeResourceMgrs(maxMemory uint64, maxFD int, connMgrHighWater int) (rm network.ResourceManager, err error) {
18+
if maxMemory == 0 {
19+
maxMemory = uint64((float64(memory.TotalMemory()) * 0.85))
20+
}
21+
if maxFD == 0 {
22+
maxFD = fd.GetNumFDs() / 2
23+
}
24+
return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(makeResourceManagerConfig(maxMemory, maxFD, connMgrHighWater)))
25+
}
26+
27+
func makeResourceManagerConfig(maxMemory uint64, maxFD int, connMgrHighWater int) (limitConfig rcmgr.ConcreteLimitConfig) {
28+
if maxMemory == 0 {
29+
maxMemory = uint64((float64(memory.TotalMemory()) * 0.85))
30+
}
31+
if maxFD == 0 {
32+
maxFD = fd.GetNumFDs() / 2
33+
}
34+
35+
maxMemoryMB := maxMemory / (1024 * 1024)
36+
37+
// At least as of 2023-01-25, it's possible to open a connection that
38+
// doesn't ask for any memory usage with the libp2p Resource Manager/Accountant
39+
// (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736).
40+
// As a result, we can't currently rely on Memory limits to full protect us.
41+
// Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed,
42+
// we take a proxy now of restricting to 1 inbound connection per MB.
43+
// Note: this is more generous than go-libp2p's default autoscaled limits which do
44+
// 64 connections per 1GB
45+
// (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ).
46+
systemConnsInbound := int(1 * maxMemoryMB)
47+
48+
partialLimits := rcmgr.PartialLimitConfig{
49+
System: rcmgr.ResourceLimits{
50+
Memory: rcmgr.LimitVal64(maxMemory),
51+
FD: rcmgr.LimitVal(maxFD),
52+
53+
Conns: rcmgr.Unlimited,
54+
ConnsInbound: rcmgr.LimitVal(systemConnsInbound),
55+
ConnsOutbound: rcmgr.Unlimited,
56+
57+
Streams: rcmgr.Unlimited,
58+
StreamsOutbound: rcmgr.Unlimited,
59+
StreamsInbound: rcmgr.Unlimited,
60+
},
61+
62+
// Transient connections won't cause any memory to be accounted for by the resource manager/accountant.
63+
// Only established connections do.
64+
// As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened.
65+
// We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope.
66+
Transient: rcmgr.ResourceLimits{
67+
Memory: rcmgr.LimitVal64(maxMemory / 4),
68+
FD: rcmgr.LimitVal(maxFD / 4),
69+
70+
Conns: rcmgr.Unlimited,
71+
ConnsInbound: rcmgr.LimitVal(systemConnsInbound / 4),
72+
ConnsOutbound: rcmgr.Unlimited,
73+
74+
Streams: rcmgr.Unlimited,
75+
StreamsInbound: rcmgr.Unlimited,
76+
StreamsOutbound: rcmgr.Unlimited,
77+
},
78+
79+
// Lets get out of the way of the allow list functionality.
80+
// If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through.
81+
AllowlistedSystem: infiniteResourceLimits,
82+
83+
AllowlistedTransient: infiniteResourceLimits,
84+
85+
// Keep it simple by not having Service, ServicePeer, Protocol, ProtocolPeer, Conn, or Stream limits.
86+
ServiceDefault: infiniteResourceLimits,
87+
88+
ServicePeerDefault: infiniteResourceLimits,
89+
90+
ProtocolDefault: infiniteResourceLimits,
91+
92+
ProtocolPeerDefault: infiniteResourceLimits,
93+
94+
Conn: infiniteResourceLimits,
95+
96+
Stream: infiniteResourceLimits,
97+
98+
// Limit the resources consumed by a peer.
99+
// This doesn't protect us against intentional DoS attacks since an attacker can easily spin up multiple peers.
100+
// We specify this limit against unintentional DoS attacks (e.g., a peer has a bug and is sending too much traffic intentionally).
101+
// In that case we want to keep that peer's resource consumption contained.
102+
// To keep this simple, we only constrain inbound connections and streams.
103+
PeerDefault: rcmgr.ResourceLimits{
104+
Memory: rcmgr.Unlimited64,
105+
FD: rcmgr.Unlimited,
106+
Conns: rcmgr.Unlimited,
107+
ConnsInbound: rcmgr.DefaultLimit,
108+
ConnsOutbound: rcmgr.Unlimited,
109+
Streams: rcmgr.Unlimited,
110+
StreamsInbound: rcmgr.DefaultLimit,
111+
StreamsOutbound: rcmgr.Unlimited,
112+
},
113+
}
114+
115+
scalingLimitConfig := rcmgr.DefaultLimits
116+
libp2p.SetDefaultServiceLimits(&scalingLimitConfig)
117+
118+
// Anything set above in partialLimits that had a value of rcmgr.DefaultLimit will be overridden.
119+
// Anything in scalingLimitConfig that wasn't defined in partialLimits above will be added (e.g., libp2p's default service limits).
120+
partialLimits = partialLimits.Build(scalingLimitConfig.Scale(int64(maxMemory), maxFD)).ToPartialLimitConfig()
121+
122+
// Simple checks to override autoscaling ensuring limits make sense versus the connmgr values.
123+
// There are ways to break this, but this should catch most problems already.
124+
// We might improve this in the future.
125+
// See: https://github.com/ipfs/kubo/issues/9545
126+
if partialLimits.System.ConnsInbound > rcmgr.DefaultLimit {
127+
maxInboundConns := int(partialLimits.System.ConnsInbound)
128+
if connmgrHighWaterTimesTwo := connMgrHighWater * 2; maxInboundConns < connmgrHighWaterTimesTwo {
129+
maxInboundConns = connmgrHighWaterTimesTwo
130+
}
131+
132+
if maxInboundConns < 800 {
133+
maxInboundConns = 800
134+
}
135+
136+
// Scale System.StreamsInbound as well, but use the existing ratio of StreamsInbound to ConnsInbound
137+
if partialLimits.System.StreamsInbound > rcmgr.DefaultLimit {
138+
partialLimits.System.StreamsInbound = rcmgr.LimitVal(int64(maxInboundConns) * int64(partialLimits.System.StreamsInbound) / int64(partialLimits.System.ConnsInbound))
139+
}
140+
partialLimits.System.ConnsInbound = rcmgr.LimitVal(maxInboundConns)
141+
}
142+
143+
logger.Infof(`
144+
145+
go-libp2p Resource Manager limits based on:
146+
- --max-memory: %s
147+
- --max-fd: %d
148+
149+
`, humanize.Bytes(maxMemory), maxFD)
150+
151+
// We already have a complete value thus pass in an empty ConcreteLimitConfig.
152+
return partialLimits.Build(rcmgr.ConcreteLimitConfig{})
153+
}

0 commit comments

Comments
 (0)