Skip to content

Commit 10d666c

Browse files
authored
op-acceptance: Add test for safe head db when EL state is deleted (#16627)
* op-acceptance: Add test for safe head db when EL state is deleted Currently skipped as this is reproducing a known issue. * op-acceptance: Review feedback. * op-acceptance: Wait for the specific peer to be connected.
1 parent 1f4438b commit 10d666c

25 files changed

+509
-41
lines changed

op-acceptance-tests/tests/base/rpc_connectivity_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ func TestRPCConnectivity(gt *testing.T) {
3939
t.Run(fmt.Sprintf("L2_Chain_%s", networkName), func(tt devtest.T) {
4040
// Get the expected chain ID from the L2Chain
4141
expectedChainID := l2Chain.ChainID().ToBig()
42-
for _, node := range l2Chain.Escape().L2ELNodes() {
43-
testL2ELNode(tt, ctx, logger, networkName, expectedChainID, dsl.NewL2ELNode(node))
42+
for _, node := range l2Chain.L2ELNodes() {
43+
testL2ELNode(tt, ctx, logger, networkName, expectedChainID, node)
4444
}
4545
})
4646
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package safeheaddb
2+
3+
import (
4+
"testing"
5+
6+
"github.com/ethereum-optimism/optimism/op-devstack/compat"
7+
"github.com/ethereum-optimism/optimism/op-devstack/presets"
8+
)
9+
10+
func TestMain(m *testing.M) {
11+
presets.DoMain(m, presets.WithSingleChainMultiNode(),
12+
presets.WithExecutionLayerSyncOnVerifiers(),
13+
presets.WithSafeDBEnabled(),
14+
// Destructive test that requiring an in-memory only geth database
15+
presets.WithCompatibleTypes(compat.SysGo),
16+
)
17+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package safeheaddb
2+
3+
import (
4+
"testing"
5+
6+
"github.com/ethereum-optimism/optimism/op-devstack/devtest"
7+
"github.com/ethereum-optimism/optimism/op-devstack/dsl"
8+
"github.com/ethereum-optimism/optimism/op-devstack/presets"
9+
"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
10+
)
11+
12+
func TestTruncateDatabaseOnResync(gt *testing.T) {
13+
t := devtest.SerialT(gt)
14+
t.Skip("Known issue: safe head db is not currently truncated when EL sync is used")
15+
sys := presets.NewSingleChainMultiNode(t)
16+
17+
dsl.CheckAll(t,
18+
sys.L2CL.AdvancedFn(types.LocalSafe, 1, 30),
19+
sys.L2CLB.AdvancedFn(types.LocalSafe, 1, 30))
20+
21+
sys.L2CLB.Matched(sys.L2CL, types.LocalSafe, 30)
22+
sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL)
23+
24+
// Stop the verifier node. Since the sysgo EL uses in-memory storage this also wipes its database.
25+
// With the EL reset to genesis, when the CL restarts it will use EL sync to resync the chain rather than
26+
// deriving it from L1.
27+
sys.L2ELB.Stop()
28+
sys.L2CLB.Stop()
29+
30+
sys.L2CL.Advanced(types.LocalSafe, 3, 30)
31+
32+
sys.L2ELB.Start()
33+
sys.L2CLB.Start()
34+
sys.L2ELB.PeerWith(sys.L2EL)
35+
36+
sys.L2CLB.Matched(sys.L2CL, types.LocalSafe, 30)
37+
sys.L2CLB.Advanced(types.LocalSafe, 1, 30) // At least one safe head db update after resync
38+
39+
sys.L2CLB.VerifySafeHeadDatabaseMatches(sys.L2CL)
40+
}

op-devstack/dsl/l2_cl.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/ethereum-optimism/optimism/op-devstack/stack"
1010
"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
11+
"github.com/ethereum-optimism/optimism/op-node/node/safedb"
1112
"github.com/ethereum-optimism/optimism/op-service/apis"
1213
"github.com/ethereum-optimism/optimism/op-service/eth"
1314
"github.com/ethereum-optimism/optimism/op-service/retry"
@@ -31,6 +32,10 @@ func NewL2CLNode(inner stack.L2CLNode, control stack.ControlPlane) *L2CLNode {
3132
}
3233
}
3334

35+
func (cl *L2CLNode) ID() stack.L2CLNodeID {
36+
return cl.inner.ID()
37+
}
38+
3439
func (cl *L2CLNode) String() string {
3540
return cl.inner.ID().String()
3641
}
@@ -119,6 +124,16 @@ func (cl *L2CLNode) ChainID() eth.ChainID {
119124
return cl.inner.ID().ChainID()
120125
}
121126

127+
func (cl *L2CLNode) AwaitMinL1Processed(minL1 uint64) {
128+
ctx, cancel := context.WithTimeout(cl.ctx, DefaultTimeout)
129+
defer cancel()
130+
// Wait for CurrentL1 to be at least one block _past_ minL1 since CurrentL1 may not yet be fully processed.
131+
err := wait.For(ctx, 1*time.Second, func() (bool, error) {
132+
return cl.SyncStatus().CurrentL1.Number > minL1, nil
133+
})
134+
cl.require.NoErrorf(err, "CurrentL1 did not reach %v", minL1+1)
135+
}
136+
122137
// AdvancedFn returns a lambda that checks the L2CL chain head with given safety level advanced more than delta block number
123138
// Composable with other lambdas to wait in parallel
124139
func (cl *L2CLNode) AdvancedFn(lvl types.SafetyLevel, delta uint64, attempts int) CheckFunc {
@@ -237,6 +252,15 @@ func (cl *L2CLNode) ChainSyncStatus(chainID eth.ChainID, lvl types.SafetyLevel)
237252
return cl.HeadBlockRef(lvl).ID()
238253
}
239254

255+
func (cl *L2CLNode) safeHeadAtL1Block(l1BlockNum uint64) *eth.SafeHeadResponse {
256+
resp, err := cl.inner.RollupAPI().SafeHeadAtL1Block(cl.ctx, l1BlockNum)
257+
if errors.Is(err, safedb.ErrNotFound) {
258+
return nil
259+
}
260+
cl.require.NoErrorf(err, "failed to get safe head at l1 block %v", l1BlockNum)
261+
return resp
262+
}
263+
240264
// LaggedFn returns a lambda that checks the L2CL chain head with given safety level is lagged with the reference chain sync status provider
241265
// Composable with other lambdas to wait in parallel
242266
func (cl *L2CLNode) LaggedFn(refNode SyncStatusProvider, lvl types.SafetyLevel, attempts int, allowMatch bool) CheckFunc {
@@ -291,3 +315,11 @@ func (cl *L2CLNode) ConnectPeer(peer *L2CLNode) {
291315
})
292316
cl.require.NoError(err, "failed to connect peer")
293317
}
318+
319+
func (cl *L2CLNode) VerifySafeHeadDatabaseMatches(sourceOfTruth *L2CLNode) {
320+
l1Block := cl.SyncStatus().CurrentL1.Number
321+
cl.log.Info("Verifying safe head database matches", "maxL1Block", l1Block)
322+
cl.AwaitMinL1Processed(l1Block) // Ensure this block is fully processed before checking safe head db
323+
sourceOfTruth.AwaitMinL1Processed(l1Block)
324+
checkSafeHeadConsistent(cl.t, l1Block, cl, sourceOfTruth)
325+
}

op-devstack/dsl/l2_el.go

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"time"
88

99
"github.com/ethereum-optimism/optimism/op-devstack/stack"
10+
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
1011
"github.com/ethereum-optimism/optimism/op-service/eth"
1112
"github.com/ethereum-optimism/optimism/op-service/retry"
1213
"github.com/ethereum/go-ethereum/common"
@@ -17,14 +18,16 @@ var emptyHash = common.Hash{}
1718
// L2ELNode wraps a stack.L2ELNode interface for DSL operations
1819
type L2ELNode struct {
1920
*elNode
20-
inner stack.L2ELNode
21+
inner stack.L2ELNode
22+
control stack.ControlPlane
2123
}
2224

2325
// NewL2ELNode creates a new L2ELNode DSL wrapper
24-
func NewL2ELNode(inner stack.L2ELNode) *L2ELNode {
26+
func NewL2ELNode(inner stack.L2ELNode, control stack.ControlPlane) *L2ELNode {
2527
return &L2ELNode{
26-
elNode: newELNode(commonFromT(inner.T()), inner),
27-
inner: inner,
28+
elNode: newELNode(commonFromT(inner.T()), inner),
29+
inner: inner,
30+
control: control,
2831
}
2932
}
3033

@@ -37,6 +40,10 @@ func (el *L2ELNode) Escape() stack.L2ELNode {
3740
return el.inner
3841
}
3942

43+
func (el *L2ELNode) ID() stack.L2ELNodeID {
44+
return el.inner.ID()
45+
}
46+
4047
func (el *L2ELNode) BlockRefByLabel(label eth.BlockLabel) eth.L2BlockRef {
4148
ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout)
4249
defer cancel()
@@ -138,3 +145,16 @@ func (el *L2ELNode) ReorgTriggered(target eth.L2BlockRef, attempts int) {
138145
func (el *L2ELNode) TransactionTimeout() time.Duration {
139146
return el.inner.TransactionTimeout()
140147
}
148+
149+
func (el *L2ELNode) Stop() {
150+
el.log.Info("Stopping", "id", el.inner.ID())
151+
el.control.L2ELNodeState(el.inner.ID(), stack.Stop)
152+
}
153+
154+
func (el *L2ELNode) Start() {
155+
el.control.L2ELNodeState(el.inner.ID(), stack.Start)
156+
}
157+
158+
func (el *L2ELNode) PeerWith(peer *L2ELNode) {
159+
sysgo.ConnectP2P(el.ctx, el.require, el.inner.L2EthClient().RPC(), peer.inner.L2EthClient().RPC())
160+
}

op-devstack/dsl/l2_network.go

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,16 @@ import (
1919
// L2Network wraps a stack.L2Network interface for DSL operations
2020
type L2Network struct {
2121
commonImpl
22-
inner stack.L2Network
22+
inner stack.L2Network
23+
control stack.ControlPlane
2324
}
2425

2526
// NewL2Network creates a new L2Network DSL wrapper
26-
func NewL2Network(inner stack.L2Network) *L2Network {
27+
func NewL2Network(inner stack.L2Network, control stack.ControlPlane) *L2Network {
2728
return &L2Network{
2829
commonImpl: commonFromT(inner.T()),
2930
inner: inner,
31+
control: control,
3032
}
3133
}
3234

@@ -43,6 +45,15 @@ func (n *L2Network) Escape() stack.L2Network {
4345
return n.inner
4446
}
4547

48+
func (n *L2Network) L2ELNodes() []*L2ELNode {
49+
innerNodes := n.inner.L2ELNodes()
50+
nodes := make([]*L2ELNode, len(innerNodes))
51+
for i, inner := range innerNodes {
52+
nodes[i] = NewL2ELNode(inner, n.control)
53+
}
54+
return nodes
55+
}
56+
4657
func (n *L2Network) CatchUpTo(o *L2Network) {
4758
this := n.inner.L2ELNode(match.FirstL2EL)
4859
other := o.inner.L2ELNode(match.FirstL2EL)
@@ -70,18 +81,18 @@ func (n *L2Network) CatchUpTo(o *L2Network) {
7081
}
7182

7283
func (n *L2Network) WaitForBlock() eth.BlockRef {
73-
return NewL2ELNode(n.inner.L2ELNode(match.FirstL2EL)).WaitForBlock()
84+
return NewL2ELNode(n.inner.L2ELNode(match.FirstL2EL), n.control).WaitForBlock()
7485
}
7586

7687
func (n *L2Network) PublicRPC() *L2ELNode {
7788
if proxyds := match.Proxyd.Match(n.Escape().L2ELNodes()); len(proxyds) > 0 {
7889
n.log.Info("PublicRPC - Using proxyd", "network", n.String())
79-
return NewL2ELNode(proxyds[0])
90+
return NewL2ELNode(proxyds[0], n.control)
8091
}
8192

8293
n.log.Info("PublicRPC - Using fallback instead of proxyd", "network", n.String())
8394
// Fallback since sysgo doesn't have proxyd support at the moment, and may never get it.
84-
return NewL2ELNode(n.inner.L2ELNode(match.FirstL2EL))
95+
return NewL2ELNode(n.inner.L2ELNode(match.FirstL2EL), n.control)
8596
}
8697

8798
// PrintChain is used for testing/debugging, it prints the blockchain hashes and parent hashes to logs, which is useful when developing reorg tests

op-devstack/dsl/safedb.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package dsl
2+
3+
import (
4+
"github.com/ethereum-optimism/optimism/op-service/eth"
5+
"github.com/ethereum-optimism/optimism/op-service/testreq"
6+
)
7+
8+
type safeHeadDBProvider interface {
9+
safeHeadAtL1Block(l1BlockNum uint64) *eth.SafeHeadResponse
10+
}
11+
12+
func checkSafeHeadConsistent(t testreq.TestingT, maxL1BlockNum uint64, checkNode, sourceOfTruth safeHeadDBProvider) {
13+
require := testreq.New(t)
14+
l1BlockNum := maxL1BlockNum
15+
matchedSomething := false
16+
for {
17+
18+
actual := checkNode.safeHeadAtL1Block(l1BlockNum)
19+
if actual == nil {
20+
// No further safe head data available
21+
// Stop iterating as long as we found _some_ data
22+
require.Truef(matchedSomething, "no safe head data available at L1 block %v", l1BlockNum)
23+
return
24+
}
25+
26+
expected := sourceOfTruth.safeHeadAtL1Block(l1BlockNum)
27+
require.Equalf(expected, actual, "Mismatched safe head data at l1 block %v", l1BlockNum)
28+
if actual.L1Block.Number == 0 {
29+
return // Reached L1 and L2 genesis.
30+
}
31+
l1BlockNum = actual.L1Block.Number - 1
32+
matchedSomething = true
33+
}
34+
}

op-devstack/presets/cl_config.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package presets
2+
3+
import (
4+
"github.com/ethereum-optimism/optimism/op-devstack/devtest"
5+
"github.com/ethereum-optimism/optimism/op-devstack/stack"
6+
"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
7+
"github.com/ethereum-optimism/optimism/op-node/config"
8+
"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
9+
)
10+
11+
func WithExecutionLayerSyncOnVerifiers() stack.CommonOption {
12+
return stack.MakeCommon(
13+
sysgo.WithL2CLOption(func(_ devtest.P, id stack.L2CLNodeID, cfg *config.Config) {
14+
// Can't enable ELSync on the sequencer or it will never start sequencing because
15+
// ELSync needs to receive gossip from the sequencer to drive the sync
16+
if !cfg.Driver.SequencerEnabled {
17+
cfg.Sync.SyncMode = sync.ELSync
18+
}
19+
}))
20+
}
21+
22+
func WithSafeDBEnabled() stack.CommonOption {
23+
return stack.MakeCommon(
24+
sysgo.WithL2CLOption(func(p devtest.P, _ stack.L2CLNodeID, cfg *config.Config) {
25+
cfg.SafeDBPath = p.TempDir()
26+
}))
27+
}

op-devstack/presets/flashblocks.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func NewSimpleFlashblocks(t devtest.T) *SimpleFlashblocks {
4949
for _, chain := range chains {
5050
chainMatcher := match.L2ChainById(chain.ID())
5151
l2 := system.L2Network(match.Assume(t, chainMatcher))
52-
firstELNode := dsl.NewL2ELNode(l2.L2ELNode(match.FirstL2EL))
52+
firstELNode := dsl.NewL2ELNode(l2.L2ELNode(match.FirstL2EL), orch.ControlPlane())
5353
firstFaucet := dsl.NewFaucet(l2.Faucet(match.Assume(t, match.FirstFaucet)))
5454

5555
conductorSets[chain.ID()] = dsl.NewConductorSet(l2.Conductors())

op-devstack/presets/interop.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ func NewSingleChainInterop(t devtest.T) *SingleChainInterop {
6565
ControlPlane: orch.ControlPlane(),
6666
L1Network: dsl.NewL1Network(l1Net),
6767
L1EL: dsl.NewL1ELNode(l1Net.L1ELNode(match.Assume(t, match.FirstL1EL))),
68-
L2ChainA: dsl.NewL2Network(l2A),
69-
L2ELA: dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.FirstL2EL))),
68+
L2ChainA: dsl.NewL2Network(l2A, orch.ControlPlane()),
69+
L2ELA: dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.FirstL2EL)), orch.ControlPlane()),
7070
L2CLA: dsl.NewL2CLNode(l2A.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane()),
7171
Wallet: dsl.NewHDWallet(t, devkeys.TestMnemonic, 30),
7272
FaucetA: dsl.NewFaucet(l2A.Faucet(match.Assume(t, match.FirstFaucet))),
@@ -155,8 +155,8 @@ func NewSimpleInterop(t devtest.T) *SimpleInterop {
155155
l2B := singleChain.system.L2Network(match.Assume(t, match.L2ChainB))
156156
out := &SimpleInterop{
157157
SingleChainInterop: *singleChain,
158-
L2ChainB: dsl.NewL2Network(l2B),
159-
L2ELB: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.FirstL2EL))),
158+
L2ChainB: dsl.NewL2Network(l2B, orch.ControlPlane()),
159+
L2ELB: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.FirstL2EL)), orch.ControlPlane()),
160160
L2CLB: dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.FirstL2CL)), orch.ControlPlane()),
161161
FaucetB: dsl.NewFaucet(l2B.Faucet(match.Assume(t, match.FirstFaucet))),
162162
L2BatcherB: dsl.NewL2Batcher(l2B.L2Batcher(match.Assume(t, match.FirstL2Batcher))),
@@ -245,9 +245,9 @@ func NewMultiSupervisorInterop(t devtest.T) *MultiSupervisorInterop {
245245
out := &MultiSupervisorInterop{
246246
SimpleInterop: *simpleInterop,
247247
SupervisorSecondary: dsl.NewSupervisor(simpleInterop.system.Supervisor(match.Assume(t, match.SecondSupervisor)), orch.ControlPlane()),
248-
L2ELA2: dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.SecondL2EL))),
248+
L2ELA2: dsl.NewL2ELNode(l2A.L2ELNode(match.Assume(t, match.SecondL2EL)), orch.ControlPlane()),
249249
L2CLA2: dsl.NewL2CLNode(l2A.L2CLNode(match.Assume(t, match.SecondL2CL)), orch.ControlPlane()),
250-
L2ELB2: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.SecondL2EL))),
250+
L2ELB2: dsl.NewL2ELNode(l2B.L2ELNode(match.Assume(t, match.SecondL2EL)), orch.ControlPlane()),
251251
L2CLB2: dsl.NewL2CLNode(l2B.L2CLNode(match.Assume(t, match.SecondL2CL)), orch.ControlPlane()),
252252
}
253253
return out

0 commit comments

Comments
 (0)