rouggy/OpsLog
0
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 67 Wiki Activity

perf(cluster): cache the worked-index and bound the spot-status cache (RBN firehose)

Browse Source 
A user on a slow PC with RBN saw OpsLog at 94% CPU and 8.5 GB RAM (Logger32: 3%
/ 34 MB on the same feeds). Two runaway costs under the spot firehose:

- ClusterSpotStatuses re-scanned the ENTIRE logbook (5-6 full-table maps) on
  every 50 ms spot batch — ~20×/second — its "one scan regardless of batch" doc
  was untrue. On a big log that's millions of row-scans/second → the pegged CPU.
  Now cached in clusterStatusCache (an immutable snapshot), rebuilt only when the
  logbook changes (noteWorked on a single log, invalidateAwardStats on bulk), so
  it's one scan per logged QSO instead of per batch.

- The frontend spotStatus map had no cap: one entry per call|band|mode ever seen,
  and RBN produces thousands of unique calls/hour → unbounded growth to GBs, plus
  a full {...prev} copy 20×/second. Now pruned back to the live (SPOTS_CAP=1000)
  spots once it drifts past 2×, with a cheap same-reference bail-out otherwise.
This commit is contained in:
rouggy
2026-08-07 00:08:24 +02:00
parent e2bfe73bdb
commit 9a72afd467
3 changed files with 115 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.go
+95 -34
View File
@@ -571,7 +571,15 @@ type App struct {
// Loaded once, appended to on each log, rebuilt after bulk changes.
wcbm map[string]struct{}
wcbmMu sync.RWMutex
pota *pota.Cache
// clusterStatusIdx caches the whole-logbook maps ClusterSpotStatuses colours
// spots against (worked entities/calls/counties/POTA/prefixes). Building them
// per spot batch re-scanned the entire logbook ~20×/second under an RBN
// firehose — the dominant CPU cost on a large log. Built lazily, treated as an
// immutable snapshot, and dropped on any logbook change (invalidateAwardStats)
// or when a setting that shapes the maps flips.
clusterStatusIdx *clusterStatusCache
clusterStatusMu sync.Mutex
pota *pota.Cache
uls *uls.Store // US callsign→county/grid (offline FCC ULS), lazily opened
awardRefs *awardref.Repo
qslTemplates *qslcard.Repo
@@ -4254,6 +4262,11 @@ func (a *App) invalidateAwardStats() {
a.awardSnap = nil
a.awardSnapRev = ""
a.awardSnapMu.Unlock()
// Drop the cluster worked-index snapshot too, so the Cluster tab's NEW/WORKED
// colouring reflects the change on the next spot batch (it rebuilds lazily).
a.clusterStatusMu.Lock()
a.clusterStatusIdx = nil
a.clusterStatusMu.Unlock()
// Bulk QSO changes (import, delete, bulk edit) also land here — refresh the
// worked-index so alert "needed" checks stay accurate. Async: never block the
// mutation, and it's a single lightweight query.
@@ -7775,6 +7788,11 @@ func (a *App) noteWorked(call, band, mode string) {
}
a.wcbm[wcbmKey(call, band, mode)] = struct{}{}
a.wcbmMu.Unlock()
// The cluster worked-index snapshot is now stale (this call/slot just became
// worked) — drop it so the next spot batch recolours with the new QSO.
a.clusterStatusMu.Lock()
a.clusterStatusIdx = nil
a.clusterStatusMu.Unlock()
}
// isWorkedBandMode reports whether this exact call+band+mode is in the log,
@@ -16086,18 +16104,46 @@ type SpotStatus struct {
Pfx string `json:"pfx,omitempty"`
}
// ClusterSpotStatuses takes a batch of spots and returns slot status for
// each. Used by the Cluster tab to color rows (NEW / NEW BAND / NEW SLOT
// / WORKED). One cty.dat lookup + one DB scan, regardless of batch size.
//
// Mode handling: when the caller passes an empty Mode (cluster comment
// was ambiguous and the frontend couldn't infer) we degrade gracefully
// to band-only — saying "worked" rather than wrongly flagging "new-slot"
// just because we don't know the mode.
func (a *App) ClusterSpotStatuses(spots []SpotQuery) []SpotStatus {
out := make([]SpotStatus, len(spots))
// clusterStatusCache holds the whole-logbook maps ClusterSpotStatuses colours
// spots against. Rebuilding them per spot batch re-scanned the entire logbook
// ~20×/second under an RBN firehose — the dominant CPU cost on a large log. This
// is an immutable snapshot: once built its maps are never mutated, so a batch
// that already holds the pointer keeps reading valid (stale-by-one-log) data
// while a newer snapshot is being built. See clusterStatusMaps.
type clusterStatusCache struct {
entities map[int]*qso.EntitySlot
workedCalls map[string]struct{}
workedCallSlots map[string]struct{} // nil unless the "same slot" option is on
workedCounties map[string]struct{}
workedPOTA map[string]struct{}
workedPfx map[string]struct{}
normMode func(string) string // nil unless digital-mode grouping is on
groupDigital bool // settings the maps were built under —
sameSlot bool // a change rebuilds the snapshot
}
// clusterStatusMaps returns the cached worked-index snapshot, building it once
// per logbook change (invalidated by invalidateAwardStats) or when a setting
// that shapes the maps flips. This turns the per-batch full-logbook scans into
// one scan per logged QSO — the fix for the RBN-firehose CPU pegging.
func (a *App) clusterStatusMaps() *clusterStatusCache {
groupDigital := a.groupDigitalSlots()
sameSlot := a.clusterWorkedSameSlot()
a.clusterStatusMu.Lock()
defer a.clusterStatusMu.Unlock()
if c := a.clusterStatusIdx; c != nil && c.groupDigital == groupDigital && c.sameSlot == sameSlot {
return c
}
c := &clusterStatusCache{groupDigital: groupDigital, sameSlot: sameSlot}
if a.qso == nil {
return out
a.clusterStatusIdx = c
return c
}
// Optional digital-mode grouping (Settings → General): with it on, FT8/FT4/
// RTTY… all count as ONE "DIG" mode, so an FT4 spot on a band where FT8 was
// worked shows "worked", not "new-slot" — DXCC-style mode classes.
if groupDigital {
c.normMode = qso.GroupDigitalMode
}
// Compare by DXCC entity NUMBER, not name. For each logged QSO the key is
// its stored DXCC if present (the authoritative value set at log time, incl.
@@ -16120,43 +16166,58 @@ func (a *App) ClusterSpotStatuses(spots []SpotQuery) []SpotStatus {
}
return 0
}
// Optional digital-mode grouping (Settings → General): with it on, FT8/FT4/
// RTTY… all count as ONE "DIG" mode, so an FT4 spot on a band where FT8 was
// worked shows "worked", not "new-slot" — DXCC-style mode classes.
var normMode func(string) string
if a.groupDigitalSlots() {
normMode = qso.GroupDigitalMode
}
entities, err := a.qso.EntitySlotMap(a.ctx, keyFor, normMode)
if err != nil {
return out
}
c.entities, _ = a.qso.EntitySlotMap(a.ctx, keyFor, c.normMode)
// Per-call worked set — separate from the entity check so we can flag
// "I've already QSO'd this exact station" even when the band/mode
// makes the entity check say "new-band" or "new-slot".
workedCalls, _ := a.qso.WorkedCallsigns(a.ctx)
c.workedCalls, _ = a.qso.WorkedCallsigns(a.ctx)
// "Already worked only on the same slot" option (Settings → DX Cluster): the
// WORKED-call flag then needs the SAME band and mode (digital-grouped through
// the same normMode when that option is on) rather than the call anywhere.
sameSlot := a.clusterWorkedSameSlot()
var workedCallSlots map[string]struct{}
if sameSlot {
workedCallSlots, _ = a.qso.WorkedCallSlotKeys(a.ctx, normMode)
c.workedCallSlots, _ = a.qso.WorkedCallSlotKeys(a.ctx, c.normMode)
}
// Orthogonal dimensions: worked US counties (for the ULS callsign→county
// lookup) and worked POTA parks. Both built once per batch.
workedCounties, _ := a.qso.WorkedCountyKeys(a.ctx, award.USCountyKey)
workedPOTA, _ := a.qso.WorkedPOTARefs(a.ctx)
// lookup) and worked POTA parks.
c.workedCounties, _ = a.qso.WorkedCountyKeys(a.ctx, award.USCountyKey)
c.workedPOTA, _ = a.qso.WorkedPOTARefs(a.ctx)
// Worked WPX prefixes, derived from the callsigns we already loaded — no
// extra query. Derived rather than read from the stored PFX column: that
// column is only filled when an import supplied it, and deriving keeps this
// in step with the WPX award, which does the same thing.
workedPfx := make(map[string]struct{}, len(workedCalls))
for c := range workedCalls {
if p := award.WPXPrefix(c); p != "" {
workedPfx[p] = struct{}{}
c.workedPfx = make(map[string]struct{}, len(c.workedCalls))
for call := range c.workedCalls {
if p := award.WPXPrefix(call); p != "" {
c.workedPfx[p] = struct{}{}
}
}
a.clusterStatusIdx = c
return c
}
// ClusterSpotStatuses takes a batch of spots and returns slot status for
// each. Used by the Cluster tab to color rows (NEW / NEW BAND / NEW SLOT
// / WORKED). Reads the cached worked-index snapshot (clusterStatusMaps) so a
// spot batch never re-scans the logbook — critical under an RBN firehose.
//
// Mode handling: when the caller passes an empty Mode (cluster comment
// was ambiguous and the frontend couldn't infer) we degrade gracefully
// to band-only — saying "worked" rather than wrongly flagging "new-slot"
// just because we don't know the mode.
func (a *App) ClusterSpotStatuses(spots []SpotQuery) []SpotStatus {
out := make([]SpotStatus, len(spots))
if a.qso == nil {
return out
}
idx := a.clusterStatusMaps()
entities := idx.entities
workedCalls := idx.workedCalls
workedCallSlots := idx.workedCallSlots
workedCounties := idx.workedCounties
workedPOTA := idx.workedPOTA
workedPfx := idx.workedPfx
normMode := idx.normMode
sameSlot := idx.sameSlot
for i, q := range spots {
out[i] = SpotStatus{
Call: q.Call,