fix: a quiet cluster was treated as a dead one and reconnected
Reported by an operator whose node sends little: nothing for two minutes, then a reconnect — losing the login, the filters, and any spot that arrived during the gap. The read had a 120 s deadline and ANY error ended the session, a timeout included. That reads a silence as a verdict on the link. It is not: a cluster on a dead band legitimately says nothing for minutes. Only a real error ends a session now. A genuinely dead peer is still caught, and by the mechanism meant for it — TCP keepalive probes an idle connection and its failure surfaces as an error on Read, not as a timeout. The dialer sets it explicitly rather than inheriting a default, since it is now what the design depends on. One trap the advice this came from did not mention: ReadString returns the bytes it HAD read along with the timeout. Discarding them would lose the first half of any spot line straddling a deadline, so the partial is carried over. Tested: a listener that goes silent past the deadline and then sends a spot — the spot arrives on the ORIGINAL connection, and the listener never accepts a second one. The tick is a var so that test runs in four seconds instead of sixty; a slow test is a test that gets skipped.
This commit is contained in:
@@ -11,6 +11,7 @@ package cluster
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
@@ -346,12 +347,29 @@ func (s *session) run() {
|
||||
}
|
||||
}
|
||||
|
||||
// idleTick is how often the read is interrupted so the loop can notice a stop
|
||||
// request. It is NOT a liveness test — see the read loop.
|
||||
//
|
||||
// A var, not a const, so the test that proves a silent node survives can shorten
|
||||
// it: at 30 s that test spends a minute doing nothing, and a slow test is a test
|
||||
// that gets skipped.
|
||||
var idleTick = 30 * time.Second
|
||||
|
||||
// quietNotice is the silence after which the log says so, once.
|
||||
const quietNotice = 10 * time.Minute
|
||||
|
||||
// runOnce dials, optionally logs in, sends init commands, parses spots.
|
||||
// Returns the moment we marked the link "connected" (zero if dial failed)
|
||||
// and the error that ended the session (nil if stopCh).
|
||||
func (s *session) runOnce() (time.Time, error) {
|
||||
addr := net.JoinHostPort(s.cfg.Host, fmt.Sprintf("%d", s.cfg.Port)) // IPv6-safe
|
||||
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
|
||||
// KeepAlive is set explicitly rather than left to the package default: it is
|
||||
// what actually detects a dead peer here, so it should be visible in the
|
||||
// code that depends on it. The OS probes an idle connection and a genuine
|
||||
// failure surfaces as an error on Read — which is the only thing that ends
|
||||
// a session below.
|
||||
d := net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}
|
||||
conn, err := d.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("dial %s: %w", addr, err)
|
||||
}
|
||||
@@ -423,6 +441,8 @@ func (s *session) runOnce() (time.Time, error) {
|
||||
}
|
||||
|
||||
var connectedAt time.Time
|
||||
var quiet time.Duration // how long the node has said nothing
|
||||
var pending string // a line cut in half by a read deadline
|
||||
rd := bufio.NewReader(conn)
|
||||
for {
|
||||
select {
|
||||
@@ -431,11 +451,37 @@ func (s *session) runOnce() (time.Time, error) {
|
||||
default:
|
||||
}
|
||||
|
||||
_ = conn.SetReadDeadline(time.Now().Add(120 * time.Second))
|
||||
line, err := rd.ReadString('\n')
|
||||
// The deadline exists to keep this loop responsive to stopCh, NOT to
|
||||
// judge the link. A quiet node is a quiet node: some clusters send
|
||||
// nothing for minutes on a dead band, and tearing the socket down over
|
||||
// it produced a reconnect every two minutes — which loses the login,
|
||||
// the filters, and any spot that arrived during the gap.
|
||||
//
|
||||
// Only a REAL error ends the session. A dead peer still gets caught:
|
||||
// TCP keepalive probes an idle connection and its failure arrives here
|
||||
// as an error, not as a timeout.
|
||||
_ = conn.SetReadDeadline(time.Now().Add(idleTick))
|
||||
chunk, err := rd.ReadString('\n')
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
// ReadString hands back what it HAD read along with the timeout.
|
||||
// Keep it: a spot line straddling the deadline would otherwise lose
|
||||
// its first half and arrive as nonsense, or vanish entirely.
|
||||
pending += chunk
|
||||
quiet += idleTick
|
||||
// Said once at the first long silence, so a genuinely mute node is
|
||||
// visible without a line every tick.
|
||||
if quiet == quietNotice {
|
||||
applog.Printf("cluster[%s] no traffic for %s — still connected", s.cfg.Name, quiet)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return connectedAt, fmt.Errorf("read: %w", err)
|
||||
}
|
||||
quiet = 0
|
||||
line := pending + chunk
|
||||
pending = ""
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
// Strip terminal BELLs (\a) and other control bytes that some nodes
|
||||
// (e.g. F5LEN) append to spot lines — "DX de …0935Z KN04\a\a" — which
|
||||
|
||||
Reference in New Issue
Block a user