fix: a quiet cluster was treated as a dead one and reconnected

Reported by an operator whose node sends little: nothing for two minutes, then a
reconnect — losing the login, the filters, and any spot that arrived during the
gap.

The read had a 120 s deadline and ANY error ended the session, a timeout
included. That reads a silence as a verdict on the link. It is not: a cluster on
a dead band legitimately says nothing for minutes.

Only a real error ends a session now. A genuinely dead peer is still caught, and
by the mechanism meant for it — TCP keepalive probes an idle connection and its
failure surfaces as an error on Read, not as a timeout. The dialer sets it
explicitly rather than inheriting a default, since it is now what the design
depends on.

One trap the advice this came from did not mention: ReadString returns the bytes
it HAD read along with the timeout. Discarding them would lose the first half of
any spot line straddling a deadline, so the partial is carried over.

Tested: a listener that goes silent past the deadline and then sends a spot —
the spot arrives on the ORIGINAL connection, and the listener never accepts a
second one. The tick is a var so that test runs in four seconds instead of
sixty; a slow test is a test that gets skipped.
This commit is contained in:
2026-07-31 21:48:05 +02:00
parent ed67ed7fe3
commit 72fee7a090
3 changed files with 142 additions and 5 deletions
+49 -3
View File
@@ -11,6 +11,7 @@ package cluster
import (
"bufio"
"errors"
"fmt"
"net"
"regexp"
@@ -346,12 +347,29 @@ func (s *session) run() {
}
}
// idleTick is how often the read is interrupted so the loop can notice a stop
// request. It is NOT a liveness test — see the read loop.
//
// A var, not a const, so the test that proves a silent node survives can shorten
// it: at 30 s that test spends a minute doing nothing, and a slow test is a test
// that gets skipped.
var idleTick = 30 * time.Second
// quietNotice is the silence after which the log says so, once.
const quietNotice = 10 * time.Minute
// runOnce dials, optionally logs in, sends init commands, parses spots.
// Returns the moment we marked the link "connected" (zero if dial failed)
// and the error that ended the session (nil if stopCh).
func (s *session) runOnce() (time.Time, error) {
addr := net.JoinHostPort(s.cfg.Host, fmt.Sprintf("%d", s.cfg.Port)) // IPv6-safe
conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
// KeepAlive is set explicitly rather than left to the package default: it is
// what actually detects a dead peer here, so it should be visible in the
// code that depends on it. The OS probes an idle connection and a genuine
// failure surfaces as an error on Read — which is the only thing that ends
// a session below.
d := net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}
conn, err := d.Dial("tcp", addr)
if err != nil {
return time.Time{}, fmt.Errorf("dial %s: %w", addr, err)
}
@@ -423,6 +441,8 @@ func (s *session) runOnce() (time.Time, error) {
}
var connectedAt time.Time
var quiet time.Duration // how long the node has said nothing
var pending string // a line cut in half by a read deadline
rd := bufio.NewReader(conn)
for {
select {
@@ -431,11 +451,37 @@ func (s *session) runOnce() (time.Time, error) {
default:
}
_ = conn.SetReadDeadline(time.Now().Add(120 * time.Second))
line, err := rd.ReadString('\n')
// The deadline exists to keep this loop responsive to stopCh, NOT to
// judge the link. A quiet node is a quiet node: some clusters send
// nothing for minutes on a dead band, and tearing the socket down over
// it produced a reconnect every two minutes — which loses the login,
// the filters, and any spot that arrived during the gap.
//
// Only a REAL error ends the session. A dead peer still gets caught:
// TCP keepalive probes an idle connection and its failure arrives here
// as an error, not as a timeout.
_ = conn.SetReadDeadline(time.Now().Add(idleTick))
chunk, err := rd.ReadString('\n')
if err != nil {
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
// ReadString hands back what it HAD read along with the timeout.
// Keep it: a spot line straddling the deadline would otherwise lose
// its first half and arrive as nonsense, or vanish entirely.
pending += chunk
quiet += idleTick
// Said once at the first long silence, so a genuinely mute node is
// visible without a line every tick.
if quiet == quietNotice {
applog.Printf("cluster[%s] no traffic for %s — still connected", s.cfg.Name, quiet)
}
continue
}
return connectedAt, fmt.Errorf("read: %w", err)
}
quiet = 0
line := pending + chunk
pending = ""
line = strings.TrimRight(line, "\r\n")
// Strip terminal BELLs (\a) and other control bytes that some nodes
// (e.g. F5LEN) append to spot lines — "DX de …0935Z KN04\a\a" — which