feat(cluster): feed the locator store from PSK Reporter, filtered at the broker
The store shipped without its main source. Locators came only from this station's own WSJT-X decodes, which is what the whole MQTT discussion was about. PSK Reporter now feeds it through a new OnGrid callback, fired before any geographic filtering: what the store wants is "which square is this callsign in", and that is true whoever happened to hear the report. The subscription filters on the RECEIVER's square, a level the v2 topic exposes. Measured on the live feed: the four opening bands unfiltered are 83 messages a second, of which roughly one in a hundred survived the NearKm test that already existed here — the rest was received, TLS-decrypted, JSON-parsed and discarded. One ring of squares is 0.2 to 1.2 a second. By square rather than by DXCC, which was the obvious alternative: one country measured 1.2 messages a second (OH) against 72.5 (K) on a single band, because a DXCC can be a continent. By square the same measurement is 0.2 to 1.2, so the load follows distance — what the feed is actually about — and is the same for every operator. Grid chasing subscribes with the "+" band wildcard, so one subscription per square covers every band instead of one per band per square. The store gains a source column (decode | mqtt), migrated in place on an existing file.
This commit is contained in:
@@ -1,27 +1,11 @@
|
||||
// Package gridcache is the long-term callsign→grid store behind grid chasing.
|
||||
//
|
||||
// A DX-cluster line never carries the DX's locator, so a spot can only show one
|
||||
// if OpsLog learnt it elsewhere: a CQ decoded over the WSJT-X UDP link, or a
|
||||
// PSK Reporter report. Learning it is easy; the problem is that the knowledge
|
||||
// died with the process. Every restart began with an empty column that took an
|
||||
// hour of listening to fill, and everything learnt yesterday was thrown away.
|
||||
// Its own SQLite file, not a table in the settings database: that one sits
|
||||
// wherever the operator put it, often a synchronised folder, and this rewrites
|
||||
// itself every minute. Deleting the file costs a few days of listening.
|
||||
//
|
||||
// So the map is persisted. Three things follow from what it is:
|
||||
//
|
||||
// - It is a CACHE, not user data. Deleting the file costs a few days of
|
||||
// listening and nothing else, which is why it lives in its own file rather
|
||||
// than in the settings database — that one sits wherever the operator chose
|
||||
// to put it, often a synchronised folder, and a store that rewrites itself
|
||||
// every minute has no business there.
|
||||
//
|
||||
// - A callsign has ONE grid. The key is unique and the newest report wins:
|
||||
// operators move, go portable, go on expedition. A stale locator is worse
|
||||
// than none for grid chasing, because it reads as a square already worked.
|
||||
//
|
||||
// - Writes are batched. The feeds repeat themselves — the same station is
|
||||
// reported by dozens of receivers a minute — so the store accumulates
|
||||
// changes in memory and flushes them on a timer. Nothing on the ingest path
|
||||
// touches the disk.
|
||||
// A callsign has one grid and the newest report wins — a stale locator reads as
|
||||
// a square already worked, which is worse than none.
|
||||
package gridcache
|
||||
|
||||
import (
|
||||
@@ -45,11 +29,23 @@ const Retention = 2 * 365 * 24 * time.Hour
|
||||
// one transaction, short enough that a crash loses a minute of learning.
|
||||
const FlushEvery = 60 * time.Second
|
||||
|
||||
// Sources a locator can come from.
|
||||
const (
|
||||
SourceDecode = "decode" // a CQ this station's own receiver decoded
|
||||
SourceMQTT = "mqtt" // a PSK Reporter report
|
||||
)
|
||||
|
||||
// Entry is one locator and where it came from.
|
||||
type Entry struct {
|
||||
Grid string
|
||||
Source string
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
|
||||
mu sync.Mutex
|
||||
dirty map[string]string // call → grid, waiting to be written
|
||||
dirty map[string]Entry // call → what to write
|
||||
|
||||
stop chan struct{}
|
||||
stopOnce sync.Once
|
||||
@@ -69,15 +65,16 @@ func Open(path string, logf func(string, ...any)) (*Store, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gridcache: open %s: %w", path, err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS grids (
|
||||
call TEXT PRIMARY KEY,
|
||||
grid TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS grids (" +
|
||||
"call TEXT PRIMARY KEY, grid TEXT NOT NULL, updated_at INTEGER NOT NULL, " +
|
||||
"source TEXT NOT NULL DEFAULT '')"); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("gridcache: schema: %w", err)
|
||||
}
|
||||
s := &Store{db: db, dirty: map[string]string{}, stop: make(chan struct{}), logf: logf}
|
||||
// A file written before the column existed keeps its rows; this fails
|
||||
// harmlessly when the column is already there.
|
||||
db.Exec("ALTER TABLE grids ADD COLUMN source TEXT NOT NULL DEFAULT ''")
|
||||
s := &Store{db: db, dirty: map[string]Entry{}, stop: make(chan struct{}), logf: logf}
|
||||
if n, err := s.prune(); err != nil {
|
||||
s.logf("gridcache: prune failed: %v", err)
|
||||
} else if n > 0 {
|
||||
@@ -118,14 +115,14 @@ func (s *Store) LoadAll() (map[string]string, error) {
|
||||
// Put queues a locator for writing. Callers pass only what CHANGED — an
|
||||
// unchanged report is the common case by a wide margin and must not reach here,
|
||||
// or the batch would carry the whole feed instead of the news in it.
|
||||
func (s *Store) Put(call, grid string) {
|
||||
func (s *Store) Put(call, grid, source string) {
|
||||
call = strings.ToUpper(strings.TrimSpace(call))
|
||||
grid = strings.TrimSpace(grid)
|
||||
if call == "" || grid == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.dirty[call] = grid
|
||||
s.dirty[call] = Entry{Grid: grid, Source: source}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -160,7 +157,7 @@ func (s *Store) Flush() error {
|
||||
return nil
|
||||
}
|
||||
batch := s.dirty
|
||||
s.dirty = map[string]string{}
|
||||
s.dirty = map[string]Entry{}
|
||||
s.mu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
@@ -168,8 +165,9 @@ func (s *Store) Flush() error {
|
||||
s.requeue(batch)
|
||||
return err
|
||||
}
|
||||
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at`)
|
||||
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at, source) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at,
|
||||
source = excluded.source`)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
s.requeue(batch)
|
||||
@@ -177,8 +175,8 @@ func (s *Store) Flush() error {
|
||||
}
|
||||
defer st.Close()
|
||||
now := time.Now().Unix()
|
||||
for call, grid := range batch {
|
||||
if _, err := st.Exec(call, grid, now); err != nil {
|
||||
for call, e := range batch {
|
||||
if _, err := st.Exec(call, e.Grid, now, e.Source); err != nil {
|
||||
tx.Rollback()
|
||||
s.requeue(batch)
|
||||
return err
|
||||
@@ -193,17 +191,17 @@ func (s *Store) Flush() error {
|
||||
|
||||
// requeue puts a failed batch back, without overwriting anything learnt while it
|
||||
// was in flight — the newer value is the right one.
|
||||
func (s *Store) requeue(batch map[string]string) {
|
||||
func (s *Store) requeue(batch map[string]Entry) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for call, grid := range batch {
|
||||
for call, e := range batch {
|
||||
if _, newer := s.dirty[call]; !newer {
|
||||
s.dirty[call] = grid
|
||||
s.dirty[call] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pending reports how many locators are waiting to be written (for diagnostics).
|
||||
// Pending reports how many locators are waiting to be written.
|
||||
func (s *Store) Pending() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -21,8 +21,8 @@ func open(t *testing.T) (*Store, string) {
|
||||
// of after an hour of listening.
|
||||
func TestSurvivesRestart(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("F4BPO", "JN36")
|
||||
s.Put("OH5CX", "KP30")
|
||||
s.Put("F4BPO", "JN36", SourceDecode)
|
||||
s.Put("OH5CX", "KP30", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
@@ -51,11 +51,11 @@ func TestNewestReportWins(t *testing.T) {
|
||||
s, _ := open(t)
|
||||
defer s.Close()
|
||||
|
||||
s.Put("F4BPO", "JN36")
|
||||
s.Put("F4BPO", "JN36", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Put("F4BPO", "KP30") // moved
|
||||
s.Put("F4BPO", "KP30", SourceDecode) // moved
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestBatching(t *testing.T) {
|
||||
defer s.Close()
|
||||
|
||||
for _, c := range []string{"A1AA", "B2BB", "C3CC"} {
|
||||
s.Put(c, "JN36")
|
||||
s.Put(c, "JN36", SourceDecode)
|
||||
}
|
||||
if n := s.Pending(); n != 3 {
|
||||
t.Errorf("pending = %d, want 3 queued and unwritten", n)
|
||||
@@ -106,8 +106,8 @@ func TestBatching(t *testing.T) {
|
||||
// square is the one way this cache can be actively wrong rather than empty.
|
||||
func TestPruneOnOpen(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("FRESH", "JN36")
|
||||
s.Put("STALE", "IO91")
|
||||
s.Put("FRESH", "JN36", SourceDecode)
|
||||
s.Put("STALE", "IO91", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func TestPruneOnOpen(t *testing.T) {
|
||||
// trade.
|
||||
func TestCloseFlushes(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("LATE", "JN36")
|
||||
s.Put("LATE", "JN36", SourceDecode)
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user