reservation: update reservation state machine

This commit updates the reservation statemachine to
allow for locking and spending of the
initial reservation.
pull/651/head
sputn1ck 9 months ago
parent 112e612c7a
commit 89b5c00cfa
No known key found for this signature in database
GPG Key ID: 671103D881A5F0E4

@ -7,6 +7,7 @@ import (
"github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/btcutil"
"github.com/lightninglabs/loop/fsm" "github.com/lightninglabs/loop/fsm"
looprpc "github.com/lightninglabs/loop/swapserverrpc" looprpc "github.com/lightninglabs/loop/swapserverrpc"
"github.com/lightningnetwork/lnd/chainntnfs"
) )
// InitReservationContext contains the request parameters for a reservation. // InitReservationContext contains the request parameters for a reservation.
@ -21,18 +22,18 @@ type InitReservationContext struct {
// InitAction is the action that is executed when the reservation state machine // InitAction is the action that is executed when the reservation state machine
// is initialized. It creates the reservation in the database and dispatches the // is initialized. It creates the reservation in the database and dispatches the
// payment to the server. // payment to the server.
func (r *FSM) InitAction(eventCtx fsm.EventContext) fsm.EventType { func (f *FSM) InitAction(eventCtx fsm.EventContext) fsm.EventType {
// Check if the context is of the correct type. // Check if the context is of the correct type.
reservationRequest, ok := eventCtx.(*InitReservationContext) reservationRequest, ok := eventCtx.(*InitReservationContext)
if !ok { if !ok {
return r.HandleError(fsm.ErrInvalidContextType) return f.HandleError(fsm.ErrInvalidContextType)
} }
keyRes, err := r.cfg.Wallet.DeriveNextKey( keyRes, err := f.cfg.Wallet.DeriveNextKey(
r.ctx, KeyFamily, f.ctx, KeyFamily,
) )
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
// Send the client reservation details to the server. // Send the client reservation details to the server.
@ -44,9 +45,9 @@ func (r *FSM) InitAction(eventCtx fsm.EventContext) fsm.EventType {
ClientKey: keyRes.PubKey.SerializeCompressed(), ClientKey: keyRes.PubKey.SerializeCompressed(),
} }
_, err = r.cfg.ReservationClient.OpenReservation(r.ctx, request) _, err = f.cfg.ReservationClient.OpenReservation(f.ctx, request)
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
reservation, err := NewReservation( reservation, err := NewReservation(
@ -59,15 +60,15 @@ func (r *FSM) InitAction(eventCtx fsm.EventContext) fsm.EventType {
keyRes.KeyLocator, keyRes.KeyLocator,
) )
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
r.reservation = reservation f.reservation = reservation
// Create the reservation in the database. // Create the reservation in the database.
err = r.cfg.Store.CreateReservation(r.ctx, reservation) err = f.cfg.Store.CreateReservation(f.ctx, reservation)
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
return OnBroadcast return OnBroadcast
@ -76,101 +77,163 @@ func (r *FSM) InitAction(eventCtx fsm.EventContext) fsm.EventType {
// SubscribeToConfirmationAction is the action that is executed when the // SubscribeToConfirmationAction is the action that is executed when the
// reservation is waiting for confirmation. It subscribes to the confirmation // reservation is waiting for confirmation. It subscribes to the confirmation
// of the reservation transaction. // of the reservation transaction.
func (r *FSM) SubscribeToConfirmationAction(_ fsm.EventContext) fsm.EventType { func (f *FSM) SubscribeToConfirmationAction(_ fsm.EventContext) fsm.EventType {
pkscript, err := r.reservation.GetPkScript() pkscript, err := f.reservation.GetPkScript()
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
callCtx, cancel := context.WithCancel(r.ctx) callCtx, cancel := context.WithCancel(f.ctx)
defer cancel() defer cancel()
// Subscribe to the confirmation of the reservation transaction. // Subscribe to the confirmation of the reservation transaction.
log.Debugf("Subscribing to conf for reservation: %x pkscript: %x, "+ log.Debugf("Subscribing to conf for reservation: %x pkscript: %x, "+
"initiation height: %v", r.reservation.ID, pkscript, "initiation height: %v", f.reservation.ID, pkscript,
r.reservation.InitiationHeight) f.reservation.InitiationHeight)
confChan, errConfChan, err := r.cfg.ChainNotifier.RegisterConfirmationsNtfn( confChan, errConfChan, err := f.cfg.ChainNotifier.RegisterConfirmationsNtfn(
callCtx, nil, pkscript, DefaultConfTarget, callCtx, nil, pkscript, DefaultConfTarget,
r.reservation.InitiationHeight, f.reservation.InitiationHeight,
) )
if err != nil { if err != nil {
r.Errorf("unable to subscribe to conf notification: %v", err) f.Errorf("unable to subscribe to conf notification: %v", err)
return r.HandleError(err) return f.HandleError(err)
} }
blockChan, errBlockChan, err := r.cfg.ChainNotifier.RegisterBlockEpochNtfn( blockChan, errBlockChan, err := f.cfg.ChainNotifier.RegisterBlockEpochNtfn(
callCtx, callCtx,
) )
if err != nil { if err != nil {
r.Errorf("unable to subscribe to block notifications: %v", err) f.Errorf("unable to subscribe to block notifications: %v", err)
return r.HandleError(err) return f.HandleError(err)
} }
// We'll now wait for the confirmation of the reservation transaction. // We'll now wait for the confirmation of the reservation transaction.
for { for {
select { select {
case err := <-errConfChan: case err := <-errConfChan:
r.Errorf("conf subscription error: %v", err) f.Errorf("conf subscription error: %v", err)
return r.HandleError(err) return f.HandleError(err)
case err := <-errBlockChan: case err := <-errBlockChan:
r.Errorf("block subscription error: %v", err) f.Errorf("block subscription error: %v", err)
return r.HandleError(err) return f.HandleError(err)
case confInfo := <-confChan: case confInfo := <-confChan:
r.Debugf("reservation confirmed: %v", confInfo) f.Debugf("confirmed in block %v", confInfo.Block)
outpoint, err := r.reservation.findReservationOutput( outpoint, err := f.reservation.findReservationOutput(
confInfo.Tx, confInfo.Tx,
) )
if err != nil { if err != nil {
return r.HandleError(err) return f.HandleError(err)
} }
r.reservation.ConfirmationHeight = confInfo.BlockHeight f.reservation.ConfirmationHeight = confInfo.BlockHeight
r.reservation.Outpoint = outpoint f.reservation.Outpoint = outpoint
return OnConfirmed return OnConfirmed
case block := <-blockChan: case block := <-blockChan:
r.Debugf("block received: %v expiry: %v", block, f.Debugf("block received: %v expiry: %v", block,
r.reservation.Expiry) f.reservation.Expiry)
if uint32(block) >= r.reservation.Expiry { if uint32(block) >= f.reservation.Expiry {
return OnTimedOut return OnTimedOut
} }
case <-r.ctx.Done(): case <-f.ctx.Done():
return fsm.NoOp return fsm.NoOp
} }
} }
} }
// ReservationConfirmedAction waits for the reservation to be either expired or // AsyncWaitForExpiredOrSweptAction waits for the reservation to be either
// waits for other actions to happen. // expired or swept. This is non-blocking and can be used to wait for the
func (r *FSM) ReservationConfirmedAction(_ fsm.EventContext) fsm.EventType { // reservation to expire while expecting other events.
blockHeightChan, errEpochChan, err := r.cfg.ChainNotifier. func (f *FSM) AsyncWaitForExpiredOrSweptAction(_ fsm.EventContext,
RegisterBlockEpochNtfn(r.ctx) ) fsm.EventType {
notifCtx, cancel := context.WithCancel(f.ctx)
blockHeightChan, errEpochChan, err := f.cfg.ChainNotifier.
RegisterBlockEpochNtfn(notifCtx)
if err != nil { if err != nil {
return r.HandleError(err) cancel()
return f.HandleError(err)
} }
pkScript, err := f.reservation.GetPkScript()
if err != nil {
cancel()
return f.HandleError(err)
}
spendChan, errSpendChan, err := f.cfg.ChainNotifier.RegisterSpendNtfn(
notifCtx, f.reservation.Outpoint, pkScript,
f.reservation.InitiationHeight,
)
if err != nil {
cancel()
return f.HandleError(err)
}
go func() {
defer cancel()
op, err := f.handleSubcriptions(
notifCtx, blockHeightChan, spendChan, errEpochChan,
errSpendChan,
)
if err != nil {
f.handleAsyncError(err)
return
}
if op == fsm.NoOp {
return
}
err = f.SendEvent(op, nil)
if err != nil {
f.Errorf("Error sending %s event: %v", op, err)
}
}()
return fsm.NoOp
}
func (f *FSM) handleSubcriptions(ctx context.Context,
blockHeightChan <-chan int32, spendChan <-chan *chainntnfs.SpendDetail,
errEpochChan <-chan error, errSpendChan <-chan error,
) (fsm.EventType, error) {
for { for {
select { select {
case err := <-errEpochChan: case err := <-errEpochChan:
return r.HandleError(err) return fsm.OnError, err
case err := <-errSpendChan:
return fsm.OnError, err
case blockHeight := <-blockHeightChan: case blockHeight := <-blockHeightChan:
expired := blockHeight >= int32(r.reservation.Expiry) expired := blockHeight >= int32(f.reservation.Expiry)
if expired {
r.Debugf("Reservation %v expired",
r.reservation.ID)
return OnTimedOut if expired {
f.Debugf("Reservation expired")
return OnTimedOut, nil
} }
case <-r.ctx.Done(): case <-spendChan:
return fsm.NoOp return OnSpent, nil
case <-ctx.Done():
return fsm.NoOp, nil
} }
} }
} }
func (f *FSM) handleAsyncError(err error) {
f.LastActionError = err
f.Errorf("Error on async action: %v", err)
err2 := f.SendEvent(fsm.OnError, err)
if err2 != nil {
f.Errorf("Error sending event: %v", err2)
}
}

@ -129,6 +129,7 @@ func TestInitReservationAction(t *testing.T) {
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc
ctxb := context.Background() ctxb := context.Background()
mockLnd := test.NewMockLnd() mockLnd := test.NewMockLnd()
mockReservationClient := new(mockReservationClient) mockReservationClient := new(mockReservationClient)
@ -223,6 +224,7 @@ func TestSubscribeToConfirmationAction(t *testing.T) {
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
chainNotifier := new(MockChainNotifier) chainNotifier := new(MockChainNotifier)
@ -304,14 +306,83 @@ func TestSubscribeToConfirmationAction(t *testing.T) {
} }
} }
// TestReservationConfirmedAction tests the ReservationConfirmedAction of the // AsyncWaitForExpiredOrSweptAction tests the AsyncWaitForExpiredOrSweptAction
// of the reservation state machine.
func TestAsyncWaitForExpiredOrSweptAction(t *testing.T) {
tests := []struct {
name string
blockErr error
spendErr error
expectedEvent fsm.EventType
}{
{
name: "noop",
expectedEvent: fsm.NoOp,
},
{
name: "block error",
blockErr: errors.New("block error"),
expectedEvent: fsm.OnError,
},
{
name: "spend error",
spendErr: errors.New("spend error"),
expectedEvent: fsm.OnError,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) { // Create a mock ChainNotifier and Reservation
chainNotifier := new(MockChainNotifier)
// Define your FSM
r := NewFSMFromReservation(
context.Background(), &Config{
ChainNotifier: chainNotifier,
},
&Reservation{
ServerPubkey: defaultPubkey,
ClientPubkey: defaultPubkey,
Expiry: defaultExpiry,
},
)
// Define the expected return values for your mocks
chainNotifier.On("RegisterBlockEpochNtfn", mock.Anything).Return(
make(chan int32), make(chan error), tc.blockErr,
)
chainNotifier.On(
"RegisterSpendNtfn", mock.Anything,
mock.Anything, mock.Anything,
).Return(
make(chan *chainntnfs.SpendDetail),
make(chan error), tc.spendErr,
)
eventType := r.AsyncWaitForExpiredOrSweptAction(nil)
// Assert that the return value is as expected
require.Equal(t, tc.expectedEvent, eventType)
})
}
}
// TesthandleSubcriptions tests the handleSubcriptions function of the
// reservation state machine. // reservation state machine.
func TestReservationConfirmedAction(t *testing.T) { func TestHandleSubcriptions(t *testing.T) {
var (
blockErr = errors.New("block error")
spendErr = errors.New("spend error")
)
tests := []struct { tests := []struct {
name string name string
blockHeight int32 blockHeight int32
blockErr error blockErr error
spendDetail *chainntnfs.SpendDetail
spendErr error
expectedEvent fsm.EventType expectedEvent fsm.EventType
expectedErr error
}{ }{
{ {
name: "expired", name: "expired",
@ -320,13 +391,25 @@ func TestReservationConfirmedAction(t *testing.T) {
}, },
{ {
name: "block error", name: "block error",
blockHeight: 0, blockErr: blockErr,
blockErr: errors.New("block error"), expectedEvent: fsm.OnError,
expectedErr: blockErr,
},
{
name: "spent",
spendDetail: &chainntnfs.SpendDetail{},
expectedEvent: OnSpent,
},
{
name: "spend error",
spendErr: spendErr,
expectedEvent: fsm.OnError, expectedEvent: fsm.OnError,
expectedErr: spendErr,
}, },
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
chainNotifier := new(MockChainNotifier) chainNotifier := new(MockChainNotifier)
@ -336,36 +419,41 @@ func TestReservationConfirmedAction(t *testing.T) {
ChainNotifier: chainNotifier, ChainNotifier: chainNotifier,
}, },
&Reservation{ &Reservation{
Expiry: defaultExpiry, ServerPubkey: defaultPubkey,
ClientPubkey: defaultPubkey,
Expiry: defaultExpiry,
}, },
) )
blockChan := make(chan int32) blockChan := make(chan int32)
blockErrChan := make(chan error) blockErrChan := make(chan error)
// Define our expected return values for the mocks. spendChan := make(chan *chainntnfs.SpendDetail)
chainNotifier.On("RegisterBlockEpochNtfn", mock.Anything).Return( spendErrChan := make(chan error)
blockChan, blockErrChan, nil,
)
go func() { go func() {
// Send the block notification.
if tc.blockHeight != 0 { if tc.blockHeight != 0 {
blockChan <- tc.blockHeight blockChan <- tc.blockHeight
} }
}()
go func() {
// Send the block notification error.
if tc.blockErr != nil { if tc.blockErr != nil {
blockErrChan <- tc.blockErr blockErrChan <- tc.blockErr
} }
if tc.spendDetail != nil {
spendChan <- tc.spendDetail
}
if tc.spendErr != nil {
spendErrChan <- tc.spendErr
}
}() }()
eventType := r.ReservationConfirmedAction(nil) eventType, err := r.handleSubcriptions(
context.Background(), blockChan, spendChan,
blockErrChan, spendErrChan,
)
require.Equal(t, tc.expectedErr, err)
require.Equal(t, tc.expectedEvent, eventType) require.Equal(t, tc.expectedEvent, eventType)
// Assert that the expected functions were called on the mocks
chainNotifier.AssertExpectations(t)
}) })
} }
} }

@ -123,6 +123,18 @@ var (
// OnRecover is the event that is triggered when the reservation FSM // OnRecover is the event that is triggered when the reservation FSM
// recovers from a restart. // recovers from a restart.
OnRecover = fsm.EventType("OnRecover") OnRecover = fsm.EventType("OnRecover")
// OnSpent is the event that is triggered when the reservation has been
// spent.
OnSpent = fsm.EventType("OnSpent")
// OnLocked is the event that is triggered when the reservation has
// been locked.
OnLocked = fsm.EventType("OnLocked")
// OnUnlocked is the event that is triggered when the reservation has
// been unlocked.
OnUnlocked = fsm.EventType("OnUnlocked")
) )
// GetReservationStates returns the statemap that defines the reservation // GetReservationStates returns the statemap that defines the reservation
@ -153,14 +165,38 @@ func (f *FSM) GetReservationStates() fsm.States {
}, },
Confirmed: fsm.State{ Confirmed: fsm.State{
Transitions: fsm.Transitions{ Transitions: fsm.Transitions{
OnTimedOut: TimedOut, OnSpent: Spent,
OnRecover: Confirmed, OnTimedOut: TimedOut,
OnRecover: Confirmed,
OnLocked: Locked,
fsm.OnError: Confirmed,
},
Action: f.AsyncWaitForExpiredOrSweptAction,
},
Locked: fsm.State{
Transitions: fsm.Transitions{
OnUnlocked: Confirmed,
OnTimedOut: TimedOut,
OnRecover: Locked,
OnSpent: Spent,
fsm.OnError: Locked,
}, },
Action: f.ReservationConfirmedAction, Action: f.AsyncWaitForExpiredOrSweptAction,
}, },
TimedOut: fsm.State{ TimedOut: fsm.State{
Transitions: fsm.Transitions{
OnTimedOut: TimedOut,
},
Action: fsm.NoOpAction, Action: fsm.NoOpAction,
}, },
Spent: fsm.State{
Transitions: fsm.Transitions{
OnSpent: Spent,
},
Action: fsm.NoOpAction,
},
Failed: fsm.State{ Failed: fsm.State{
Action: fsm.NoOpAction, Action: fsm.NoOpAction,
}, },

@ -3,6 +3,7 @@ package reservation
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"sync" "sync"
"time" "time"
@ -35,7 +36,6 @@ func NewManager(cfg *Config) *Manager {
// Run runs the reservation manager. // Run runs the reservation manager.
func (m *Manager) Run(ctx context.Context, height int32) error { func (m *Manager) Run(ctx context.Context, height int32) error {
// todo(sputn1ck): recover swaps on startup
log.Debugf("Starting reservation manager") log.Debugf("Starting reservation manager")
runCtx, cancel := context.WithCancel(ctx) runCtx, cancel := context.WithCancel(ctx)
@ -269,3 +269,54 @@ func (m *Manager) RecoverReservations(ctx context.Context) error {
func (m *Manager) GetReservations(ctx context.Context) ([]*Reservation, error) { func (m *Manager) GetReservations(ctx context.Context) ([]*Reservation, error) {
return m.cfg.Store.ListReservations(ctx) return m.cfg.Store.ListReservations(ctx)
} }
// GetReservation returns the reservation for the given id.
func (m *Manager) GetReservation(ctx context.Context, id ID) (*Reservation,
error) {
return m.cfg.Store.GetReservation(ctx, id)
}
// LockReservation locks the reservation with the given ID.
func (m *Manager) LockReservation(ctx context.Context, id ID) error {
// Try getting the reservation from the active reservations map.
m.Lock()
reservation, ok := m.activeReservations[id]
m.Unlock()
if !ok {
return fmt.Errorf("reservation not found")
}
// Try to send the lock event to the reservation.
err := reservation.SendEvent(OnLocked, nil)
if err != nil {
return err
}
return nil
}
// UnlockReservation unlocks the reservation with the given ID.
func (m *Manager) UnlockReservation(ctx context.Context, id ID) error {
// Try getting the reservation from the active reservations map.
m.Lock()
reservation, ok := m.activeReservations[id]
m.Unlock()
if !ok {
return fmt.Errorf("reservation not found")
}
// Try to send the unlock event to the reservation.
err := reservation.SendEvent(OnUnlocked, nil)
if err != nil && strings.Contains(err.Error(), "config error") {
// If the error is a config error, we can ignore it, as the
// reservation is already unlocked.
return nil
} else if err != nil {
return err
}
return nil
}

@ -33,7 +33,7 @@ func TestManager(t *testing.T) {
}() }()
// Create a new reservation. // Create a new reservation.
fsm, err := testContext.manager.newReservation( reservationFSM, err := testContext.manager.newReservation(
ctxb, uint32(testContext.mockLnd.Height), ctxb, uint32(testContext.mockLnd.Height),
&swapserverrpc.ServerReservationNotification{ &swapserverrpc.ServerReservationNotification{
ReservationId: defaultReservationId[:], ReservationId: defaultReservationId[:],
@ -45,11 +45,11 @@ func TestManager(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
// We'll expect the spendConfirmation to be sent to the server. // We'll expect the spendConfirmation to be sent to the server.
pkScript, err := fsm.reservation.GetPkScript() pkScript, err := reservationFSM.reservation.GetPkScript()
require.NoError(t, err) require.NoError(t, err)
conf := <-testContext.mockLnd.RegisterConfChannel confReg := <-testContext.mockLnd.RegisterConfChannel
require.Equal(t, conf.PkScript, pkScript) require.Equal(t, confReg.PkScript, pkScript)
confTx := &wire.MsgTx{ confTx := &wire.MsgTx{
TxOut: []*wire.TxOut{ TxOut: []*wire.TxOut{
@ -59,23 +59,39 @@ func TestManager(t *testing.T) {
}, },
} }
// We'll now confirm the spend. // We'll now confirm the spend.
conf.ConfChan <- &chainntnfs.TxConfirmation{ confReg.ConfChan <- &chainntnfs.TxConfirmation{
BlockHeight: uint32(testContext.mockLnd.Height), BlockHeight: uint32(testContext.mockLnd.Height),
Tx: confTx, Tx: confTx,
} }
// We'll now expect the reservation to be confirmed. // We'll now expect the reservation to be confirmed.
err = fsm.DefaultObserver.WaitForState(ctxb, 5*time.Second, Confirmed) err = reservationFSM.DefaultObserver.WaitForState(ctxb, 5*time.Second, Confirmed)
require.NoError(t, err) require.NoError(t, err)
// We'll now expire the reservation. // We'll now expect a spend registration.
err = testContext.mockLnd.NotifyHeight( spendReg := <-testContext.mockLnd.RegisterSpendChannel
testContext.mockLnd.Height + int32(defaultExpiry), require.Equal(t, spendReg.PkScript, pkScript)
)
go func() {
// We'll expect a second spend registration.
spendReg = <-testContext.mockLnd.RegisterSpendChannel
require.Equal(t, spendReg.PkScript, pkScript)
}()
// We'll now try to lock the reservation.
err = testContext.manager.LockReservation(ctxb, defaultReservationId)
require.NoError(t, err) require.NoError(t, err)
// We'll try to lock the reservation again, which should fail.
err = testContext.manager.LockReservation(ctxb, defaultReservationId)
require.Error(t, err)
testContext.mockLnd.SpendChannel <- &chainntnfs.SpendDetail{
SpentOutPoint: spendReg.Outpoint,
}
// We'll now expect the reservation to be expired. // We'll now expect the reservation to be expired.
err = fsm.DefaultObserver.WaitForState(ctxb, 5*time.Second, TimedOut) err = reservationFSM.DefaultObserver.WaitForState(ctxb, 5*time.Second, Spent)
require.NoError(t, err) require.NoError(t, err)
} }

@ -2,6 +2,7 @@
stateDiagram-v2 stateDiagram-v2
[*] --> Init: OnServerRequest [*] --> Init: OnServerRequest
Confirmed Confirmed
Confirmed --> SpendBroadcasted: OnSpendBroadcasted
Confirmed --> TimedOut: OnTimedOut Confirmed --> TimedOut: OnTimedOut
Confirmed --> Confirmed: OnRecover Confirmed --> Confirmed: OnRecover
Failed Failed
@ -9,6 +10,9 @@ Init
Init --> Failed: OnError Init --> Failed: OnError
Init --> WaitForConfirmation: OnBroadcast Init --> WaitForConfirmation: OnBroadcast
Init --> Failed: OnRecover Init --> Failed: OnRecover
SpendBroadcasted
SpendBroadcasted --> SpendConfirmed: OnSpendConfirmed
SpendConfirmed
TimedOut TimedOut
WaitForConfirmation WaitForConfirmation
WaitForConfirmation --> WaitForConfirmation: OnRecover WaitForConfirmation --> WaitForConfirmation: OnRecover

Loading…
Cancel
Save