From 09a9bc289990e3f377a331626b0102283d6f83f4 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 5 May 2018 05:33:29 +0200 Subject: [PATCH 1/3] Fix infinite loop in exit routine --- receive.go | 6 ++++-- send.go | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/receive.go b/receive.go index 1d8b718..dc6488f 100644 --- a/receive.go +++ b/receive.go @@ -245,9 +245,10 @@ func (device *Device) RoutineDecryption() { elem.Drop() } default: - break + goto out } } + out: logDebug.Println("Routine: decryption worker - stopped") }() logDebug.Println("Routine: decryption worker - started") @@ -317,9 +318,10 @@ func (device *Device) RoutineHandshake() { select { case <-device.queue.handshake: default: - return + goto out } } + out: logDebug.Println("Routine: handshake worker - stopped") }() diff --git a/send.go b/send.go index 7423e3b..7abe211 100644 --- a/send.go +++ b/send.go @@ -281,9 +281,10 @@ func (device *Device) RoutineEncryption() { elem.Drop() } default: - break + goto out } } + out: logDebug.Println("Routine: encryption worker - stopped") }() From a46401bbb151d8f1e662dc16b612426352138c1e Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 5 May 2018 06:00:38 +0200 Subject: [PATCH 2/3] More robust solution to close deadlock --- device.go | 31 ++++++++++++++++++++++++++++++- receive.go | 21 ++------------------- send.go | 12 +----------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/device.go b/device.go index 8b0d2a5..c714b21 100644 --- a/device.go +++ b/device.go @@ -13,6 +13,10 @@ import ( "time" ) +const ( + DeviceRoutineNumberPerCPU = 3 +) + type Device struct { isUp AtomicBool // device is (going) up isClosed AtomicBool // device is closed? (acting as guard) @@ -21,6 +25,7 @@ type Device struct { // synchronized resources (locks acquired in order) state struct { + stopping sync.WaitGroup mutex sync.Mutex changing AtomicBool current bool @@ -306,7 +311,9 @@ func NewDevice(tun TUNDevice, logger *Logger) *Device { // start workers - for i := 0; i < runtime.NumCPU(); i += 1 { + cpus := runtime.NumCPU() + device.state.stopping.Add(DeviceRoutineNumberPerCPU * cpus) + for i := 0; i < cpus; i += 1 { go device.RoutineEncryption() go device.RoutineDecryption() go device.RoutineHandshake() @@ -360,6 +367,25 @@ func (device *Device) RemoveAllPeers() { device.peers.keyMap = make(map[NoisePublicKey]*Peer) } +func (device *Device) FlushPacketQueues() { + for { + select { + case elem, ok := <-device.queue.decryption: + if ok { + elem.Drop() + } + case elem, ok := <-device.queue.encryption: + if ok { + elem.Drop() + } + case <-device.queue.handshake: + default: + return + } + } + +} + func (device *Device) Close() { if device.isClosed.Swap(true) { return @@ -376,6 +402,9 @@ func (device *Device) Close() { device.signal.stop.Broadcast() + device.state.stopping.Wait() + device.FlushPacketQueues() + device.RemoveAllPeers() device.rate.limiter.Close() diff --git a/receive.go b/receive.go index dc6488f..e323c6d 100644 --- a/receive.go +++ b/receive.go @@ -238,17 +238,7 @@ func (device *Device) RoutineDecryption() { logDebug := device.log.Debug defer func() { - for { - select { - case elem, ok := <-device.queue.decryption: - if ok { - elem.Drop() - } - default: - goto out - } - } - out: + device.state.stopping.Done() logDebug.Println("Routine: decryption worker - stopped") }() logDebug.Println("Routine: decryption worker - started") @@ -314,14 +304,7 @@ func (device *Device) RoutineHandshake() { logDebug := device.log.Debug defer func() { - for { - select { - case <-device.queue.handshake: - default: - goto out - } - } - out: + device.state.stopping.Done() logDebug.Println("Routine: handshake worker - stopped") }() diff --git a/send.go b/send.go index 7abe211..e41be83 100644 --- a/send.go +++ b/send.go @@ -274,17 +274,7 @@ func (device *Device) RoutineEncryption() { logDebug := device.log.Debug defer func() { - for { - select { - case elem, ok := <-device.queue.encryption: - if ok { - elem.Drop() - } - default: - goto out - } - } - out: + device.state.stopping.Done() logDebug.Println("Routine: encryption worker - stopped") }() From c29428b075aa4476f3f3f0b97e580c2f4b8bb683 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sat, 5 May 2018 06:09:30 +0200 Subject: [PATCH 3/3] Reorder stopping messages so that logs are coherent --- receive.go | 6 +++--- send.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/receive.go b/receive.go index e323c6d..d4245af 100644 --- a/receive.go +++ b/receive.go @@ -238,8 +238,8 @@ func (device *Device) RoutineDecryption() { logDebug := device.log.Debug defer func() { - device.state.stopping.Done() logDebug.Println("Routine: decryption worker - stopped") + device.state.stopping.Done() }() logDebug.Println("Routine: decryption worker - started") @@ -304,8 +304,8 @@ func (device *Device) RoutineHandshake() { logDebug := device.log.Debug defer func() { - device.state.stopping.Done() logDebug.Println("Routine: handshake worker - stopped") + device.state.stopping.Done() }() logDebug.Println("Routine: handshake worker - started") @@ -534,8 +534,8 @@ func (peer *Peer) RoutineSequentialReceiver() { logDebug := device.log.Debug defer func() { - peer.routines.stopping.Done() logDebug.Println(peer, ": Routine: sequential receiver - stopped") + peer.routines.stopping.Done() }() logDebug.Println(peer, ": Routine: sequential receiver - started") diff --git a/send.go b/send.go index e41be83..4ed084d 100644 --- a/send.go +++ b/send.go @@ -200,8 +200,8 @@ func (peer *Peer) RoutineNonce() { logDebug := device.log.Debug defer func() { - peer.routines.stopping.Done() logDebug.Println(peer, ": Routine: nonce worker - stopped") + peer.routines.stopping.Done() }() peer.routines.starting.Done() @@ -274,8 +274,8 @@ func (device *Device) RoutineEncryption() { logDebug := device.log.Debug defer func() { - device.state.stopping.Done() logDebug.Println("Routine: encryption worker - stopped") + device.state.stopping.Done() }() logDebug.Println("Routine: encryption worker - started") @@ -348,8 +348,8 @@ func (peer *Peer) RoutineSequentialSender() { logDebug := device.log.Debug defer func() { - peer.routines.stopping.Done() logDebug.Println(peer, ": Routine: sequential sender - stopped") + peer.routines.stopping.Done() }() logDebug.Println(peer, ": Routine: sequential sender - started")