feat(cluster): feed the locator store from PSK Reporter, filtered at the broker

The store shipped without its main source. Locators came only from this
station's own WSJT-X decodes, which is what the whole MQTT discussion was
about.

PSK Reporter now feeds it through a new OnGrid callback, fired before any
geographic filtering: what the store wants is "which square is this callsign
in", and that is true whoever happened to hear the report.

The subscription filters on the RECEIVER's square, a level the v2 topic
exposes. Measured on the live feed: the four opening bands unfiltered are 83
messages a second, of which roughly one in a hundred survived the NearKm test
that already existed here — the rest was received, TLS-decrypted, JSON-parsed
and discarded. One ring of squares is 0.2 to 1.2 a second.

By square rather than by DXCC, which was the obvious alternative: one country
measured 1.2 messages a second (OH) against 72.5 (K) on a single band, because
a DXCC can be a continent. By square the same measurement is 0.2 to 1.2, so the
load follows distance — what the feed is actually about — and is the same for
every operator.

Grid chasing subscribes with the "+" band wildcard, so one subscription per
square covers every band instead of one per band per square.

The store gains a source column (decode | mqtt), migrated in place on an
existing file.
This commit is contained in:
2026-08-12 17:16:47 +02:00
parent 82a49150d2
commit 7d33379fe1
15 changed files with 352 additions and 81 deletions
+37 -39
View File
@@ -1,27 +1,11 @@
// Package gridcache is the long-term callsign→grid store behind grid chasing.
//
// A DX-cluster line never carries the DX's locator, so a spot can only show one
// if OpsLog learnt it elsewhere: a CQ decoded over the WSJT-X UDP link, or a
// PSK Reporter report. Learning it is easy; the problem is that the knowledge
// died with the process. Every restart began with an empty column that took an
// hour of listening to fill, and everything learnt yesterday was thrown away.
// Its own SQLite file, not a table in the settings database: that one sits
// wherever the operator put it, often a synchronised folder, and this rewrites
// itself every minute. Deleting the file costs a few days of listening.
//
// So the map is persisted. Three things follow from what it is:
//
// - It is a CACHE, not user data. Deleting the file costs a few days of
// listening and nothing else, which is why it lives in its own file rather
// than in the settings database — that one sits wherever the operator chose
// to put it, often a synchronised folder, and a store that rewrites itself
// every minute has no business there.
//
// - A callsign has ONE grid. The key is unique and the newest report wins:
// operators move, go portable, go on expedition. A stale locator is worse
// than none for grid chasing, because it reads as a square already worked.
//
// - Writes are batched. The feeds repeat themselves — the same station is
// reported by dozens of receivers a minute — so the store accumulates
// changes in memory and flushes them on a timer. Nothing on the ingest path
// touches the disk.
// A callsign has one grid and the newest report wins — a stale locator reads as
// a square already worked, which is worse than none.
package gridcache
import (
@@ -45,11 +29,23 @@ const Retention = 2 * 365 * 24 * time.Hour
// one transaction, short enough that a crash loses a minute of learning.
const FlushEvery = 60 * time.Second
// Sources a locator can come from.
const (
SourceDecode = "decode" // a CQ this station's own receiver decoded
SourceMQTT = "mqtt" // a PSK Reporter report
)
// Entry is one locator and where it came from.
type Entry struct {
Grid string
Source string
}
type Store struct {
db *sql.DB
mu sync.Mutex
dirty map[string]string // call → grid, waiting to be written
dirty map[string]Entry // call → what to write
stop chan struct{}
stopOnce sync.Once
@@ -69,15 +65,16 @@ func Open(path string, logf func(string, ...any)) (*Store, error) {
if err != nil {
return nil, fmt.Errorf("gridcache: open %s: %w", path, err)
}
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS grids (
call TEXT PRIMARY KEY,
grid TEXT NOT NULL,
updated_at INTEGER NOT NULL
)`); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS grids (" +
"call TEXT PRIMARY KEY, grid TEXT NOT NULL, updated_at INTEGER NOT NULL, " +
"source TEXT NOT NULL DEFAULT '')"); err != nil {
db.Close()
return nil, fmt.Errorf("gridcache: schema: %w", err)
}
s := &Store{db: db, dirty: map[string]string{}, stop: make(chan struct{}), logf: logf}
// A file written before the column existed keeps its rows; this fails
// harmlessly when the column is already there.
db.Exec("ALTER TABLE grids ADD COLUMN source TEXT NOT NULL DEFAULT ''")
s := &Store{db: db, dirty: map[string]Entry{}, stop: make(chan struct{}), logf: logf}
if n, err := s.prune(); err != nil {
s.logf("gridcache: prune failed: %v", err)
} else if n > 0 {
@@ -118,14 +115,14 @@ func (s *Store) LoadAll() (map[string]string, error) {
// Put queues a locator for writing. Callers pass only what CHANGED — an
// unchanged report is the common case by a wide margin and must not reach here,
// or the batch would carry the whole feed instead of the news in it.
func (s *Store) Put(call, grid string) {
func (s *Store) Put(call, grid, source string) {
call = strings.ToUpper(strings.TrimSpace(call))
grid = strings.TrimSpace(grid)
if call == "" || grid == "" {
return
}
s.mu.Lock()
s.dirty[call] = grid
s.dirty[call] = Entry{Grid: grid, Source: source}
s.mu.Unlock()
}
@@ -160,7 +157,7 @@ func (s *Store) Flush() error {
return nil
}
batch := s.dirty
s.dirty = map[string]string{}
s.dirty = map[string]Entry{}
s.mu.Unlock()
tx, err := s.db.Begin()
@@ -168,8 +165,9 @@ func (s *Store) Flush() error {
s.requeue(batch)
return err
}
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at) VALUES (?, ?, ?)
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at`)
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at, source) VALUES (?, ?, ?, ?)
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at,
source = excluded.source`)
if err != nil {
tx.Rollback()
s.requeue(batch)
@@ -177,8 +175,8 @@ func (s *Store) Flush() error {
}
defer st.Close()
now := time.Now().Unix()
for call, grid := range batch {
if _, err := st.Exec(call, grid, now); err != nil {
for call, e := range batch {
if _, err := st.Exec(call, e.Grid, now, e.Source); err != nil {
tx.Rollback()
s.requeue(batch)
return err
@@ -193,17 +191,17 @@ func (s *Store) Flush() error {
// requeue puts a failed batch back, without overwriting anything learnt while it
// was in flight — the newer value is the right one.
func (s *Store) requeue(batch map[string]string) {
func (s *Store) requeue(batch map[string]Entry) {
s.mu.Lock()
defer s.mu.Unlock()
for call, grid := range batch {
for call, e := range batch {
if _, newer := s.dirty[call]; !newer {
s.dirty[call] = grid
s.dirty[call] = e
}
}
}
// Pending reports how many locators are waiting to be written (for diagnostics).
// Pending reports how many locators are waiting to be written.
func (s *Store) Pending() int {
s.mu.Lock()
defer s.mu.Unlock()
+8 -8
View File
@@ -21,8 +21,8 @@ func open(t *testing.T) (*Store, string) {
// of after an hour of listening.
func TestSurvivesRestart(t *testing.T) {
s, path := open(t)
s.Put("F4BPO", "JN36")
s.Put("OH5CX", "KP30")
s.Put("F4BPO", "JN36", SourceDecode)
s.Put("OH5CX", "KP30", SourceDecode)
if err := s.Flush(); err != nil {
t.Fatalf("flush: %v", err)
}
@@ -51,11 +51,11 @@ func TestNewestReportWins(t *testing.T) {
s, _ := open(t)
defer s.Close()
s.Put("F4BPO", "JN36")
s.Put("F4BPO", "JN36", SourceDecode)
if err := s.Flush(); err != nil {
t.Fatal(err)
}
s.Put("F4BPO", "KP30") // moved
s.Put("F4BPO", "KP30", SourceDecode) // moved
if err := s.Flush(); err != nil {
t.Fatal(err)
}
@@ -79,7 +79,7 @@ func TestBatching(t *testing.T) {
defer s.Close()
for _, c := range []string{"A1AA", "B2BB", "C3CC"} {
s.Put(c, "JN36")
s.Put(c, "JN36", SourceDecode)
}
if n := s.Pending(); n != 3 {
t.Errorf("pending = %d, want 3 queued and unwritten", n)
@@ -106,8 +106,8 @@ func TestBatching(t *testing.T) {
// square is the one way this cache can be actively wrong rather than empty.
func TestPruneOnOpen(t *testing.T) {
s, path := open(t)
s.Put("FRESH", "JN36")
s.Put("STALE", "IO91")
s.Put("FRESH", "JN36", SourceDecode)
s.Put("STALE", "IO91", SourceDecode)
if err := s.Flush(); err != nil {
t.Fatal(err)
}
@@ -137,7 +137,7 @@ func TestPruneOnOpen(t *testing.T) {
// trade.
func TestCloseFlushes(t *testing.T) {
s, path := open(t)
s.Put("LATE", "JN36")
s.Put("LATE", "JN36", SourceDecode)
if err := s.Close(); err != nil {
t.Fatalf("close: %v", err)
}