feat(cluster): feed the locator store from PSK Reporter, filtered at the broker
The store shipped without its main source. Locators came only from this station's own WSJT-X decodes, which is what the whole MQTT discussion was about. PSK Reporter now feeds it through a new OnGrid callback, fired before any geographic filtering: what the store wants is "which square is this callsign in", and that is true whoever happened to hear the report. The subscription filters on the RECEIVER's square, a level the v2 topic exposes. Measured on the live feed: the four opening bands unfiltered are 83 messages a second, of which roughly one in a hundred survived the NearKm test that already existed here — the rest was received, TLS-decrypted, JSON-parsed and discarded. One ring of squares is 0.2 to 1.2 a second. By square rather than by DXCC, which was the obvious alternative: one country measured 1.2 messages a second (OH) against 72.5 (K) on a single band, because a DXCC can be a continent. By square the same measurement is 0.2 to 1.2, so the load follows distance — what the feed is actually about — and is the same for every operator. Grid chasing subscribes with the "+" band wildcard, so one subscription per square covers every band instead of one per band per square. The store gains a source column (decode | mqtt), migrated in place on an existing file.
This commit is contained in:
@@ -67,3 +67,54 @@ func DistanceBetweenGrids(a, b string) (km float64, ok bool) {
|
||||
}
|
||||
return HaversineKm(lat1, lon1, lat2, lon2), true
|
||||
}
|
||||
|
||||
// LatLonToGrid returns the 4-character Maidenhead square for a position.
|
||||
func LatLonToGrid(lat, lon float64) string {
|
||||
lon = math.Mod(lon+180, 360)
|
||||
if lon < 0 {
|
||||
lon += 360
|
||||
}
|
||||
lat = lat + 90
|
||||
if lat < 0 {
|
||||
lat = 0
|
||||
} else if lat > 180 {
|
||||
lat = 180
|
||||
}
|
||||
return string([]byte{
|
||||
byte('A' + int(lon/20)),
|
||||
byte('A' + int(lat/10)),
|
||||
byte('0' + int(math.Mod(lon, 20)/2)),
|
||||
byte('0' + int(math.Mod(lat, 10)/1)),
|
||||
})
|
||||
}
|
||||
|
||||
// NeighbourGrids returns the square holding (lat, lon) and the ring of squares
|
||||
// around it — 9 squares for ring 1, 25 for ring 2.
|
||||
//
|
||||
// Used to filter the PSK Reporter feed at the BROKER rather than in OpsLog. A
|
||||
// square is about 111 km tall and 150 km wide at mid latitudes, so one ring is
|
||||
// roughly the 300 km "around here" the feed already meant, and the traffic that
|
||||
// used to be received and discarded is never sent.
|
||||
func NeighbourGrids(lat, lon float64, ring int) []string {
|
||||
if ring < 0 {
|
||||
ring = 0
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for dLat := -ring; dLat <= ring; dLat++ {
|
||||
for dLon := -ring; dLon <= ring; dLon++ {
|
||||
// One square step: 1° of latitude, 2° of longitude.
|
||||
la := lat + float64(dLat)
|
||||
lo := lon + float64(dLon)*2
|
||||
if la > 90 || la < -90 {
|
||||
continue // past a pole there is no square, not a wrapped one
|
||||
}
|
||||
g := LatLonToGrid(la, lo)
|
||||
if !seen[g] {
|
||||
seen[g] = true
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package geo
|
||||
|
||||
import "testing"
|
||||
|
||||
// The squares the PSK Reporter feed is filtered on. A wrong ring means either
|
||||
// receiving the world again or hearing nothing.
|
||||
func TestNeighbourGrids(t *testing.T) {
|
||||
// JN36 is around 46.5N 5.5E.
|
||||
lat, lon, ok := GridToLatLon("JN36")
|
||||
if !ok {
|
||||
t.Fatal("JN36 did not resolve")
|
||||
}
|
||||
if got := LatLonToGrid(lat, lon); got != "JN36" {
|
||||
t.Fatalf("round trip gave %q, want JN36", got)
|
||||
}
|
||||
|
||||
ring := NeighbourGrids(lat, lon, 1)
|
||||
if len(ring) != 9 {
|
||||
t.Errorf("ring 1 has %d squares, want 9: %v", len(ring), ring)
|
||||
}
|
||||
found := false
|
||||
for _, g := range ring {
|
||||
if g == "JN36" {
|
||||
found = true
|
||||
}
|
||||
if len(g) != 4 {
|
||||
t.Errorf("not a 4-character square: %q", g)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("the operator's own square is missing from %v", ring)
|
||||
}
|
||||
if n := len(NeighbourGrids(lat, lon, 0)); n != 1 {
|
||||
t.Errorf("ring 0 has %d squares, want just the operator's", n)
|
||||
}
|
||||
if n := len(NeighbourGrids(lat, lon, 2)); n != 25 {
|
||||
t.Errorf("ring 2 has %d squares, want 25", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Near a pole a step north has nowhere to go; it must be dropped, not wrapped
|
||||
// onto a square on the far side of the world.
|
||||
func TestNeighbourGridsNearThePole(t *testing.T) {
|
||||
for _, g := range NeighbourGrids(89.5, 25, 1) {
|
||||
if len(g) != 4 {
|
||||
t.Errorf("bad square near the pole: %q", g)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,11 @@
|
||||
// Package gridcache is the long-term callsign→grid store behind grid chasing.
|
||||
//
|
||||
// A DX-cluster line never carries the DX's locator, so a spot can only show one
|
||||
// if OpsLog learnt it elsewhere: a CQ decoded over the WSJT-X UDP link, or a
|
||||
// PSK Reporter report. Learning it is easy; the problem is that the knowledge
|
||||
// died with the process. Every restart began with an empty column that took an
|
||||
// hour of listening to fill, and everything learnt yesterday was thrown away.
|
||||
// Its own SQLite file, not a table in the settings database: that one sits
|
||||
// wherever the operator put it, often a synchronised folder, and this rewrites
|
||||
// itself every minute. Deleting the file costs a few days of listening.
|
||||
//
|
||||
// So the map is persisted. Three things follow from what it is:
|
||||
//
|
||||
// - It is a CACHE, not user data. Deleting the file costs a few days of
|
||||
// listening and nothing else, which is why it lives in its own file rather
|
||||
// than in the settings database — that one sits wherever the operator chose
|
||||
// to put it, often a synchronised folder, and a store that rewrites itself
|
||||
// every minute has no business there.
|
||||
//
|
||||
// - A callsign has ONE grid. The key is unique and the newest report wins:
|
||||
// operators move, go portable, go on expedition. A stale locator is worse
|
||||
// than none for grid chasing, because it reads as a square already worked.
|
||||
//
|
||||
// - Writes are batched. The feeds repeat themselves — the same station is
|
||||
// reported by dozens of receivers a minute — so the store accumulates
|
||||
// changes in memory and flushes them on a timer. Nothing on the ingest path
|
||||
// touches the disk.
|
||||
// A callsign has one grid and the newest report wins — a stale locator reads as
|
||||
// a square already worked, which is worse than none.
|
||||
package gridcache
|
||||
|
||||
import (
|
||||
@@ -45,11 +29,23 @@ const Retention = 2 * 365 * 24 * time.Hour
|
||||
// one transaction, short enough that a crash loses a minute of learning.
|
||||
const FlushEvery = 60 * time.Second
|
||||
|
||||
// Sources a locator can come from.
|
||||
const (
|
||||
SourceDecode = "decode" // a CQ this station's own receiver decoded
|
||||
SourceMQTT = "mqtt" // a PSK Reporter report
|
||||
)
|
||||
|
||||
// Entry is one locator and where it came from.
|
||||
type Entry struct {
|
||||
Grid string
|
||||
Source string
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
|
||||
mu sync.Mutex
|
||||
dirty map[string]string // call → grid, waiting to be written
|
||||
dirty map[string]Entry // call → what to write
|
||||
|
||||
stop chan struct{}
|
||||
stopOnce sync.Once
|
||||
@@ -69,15 +65,16 @@ func Open(path string, logf func(string, ...any)) (*Store, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gridcache: open %s: %w", path, err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS grids (
|
||||
call TEXT PRIMARY KEY,
|
||||
grid TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS grids (" +
|
||||
"call TEXT PRIMARY KEY, grid TEXT NOT NULL, updated_at INTEGER NOT NULL, " +
|
||||
"source TEXT NOT NULL DEFAULT '')"); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("gridcache: schema: %w", err)
|
||||
}
|
||||
s := &Store{db: db, dirty: map[string]string{}, stop: make(chan struct{}), logf: logf}
|
||||
// A file written before the column existed keeps its rows; this fails
|
||||
// harmlessly when the column is already there.
|
||||
db.Exec("ALTER TABLE grids ADD COLUMN source TEXT NOT NULL DEFAULT ''")
|
||||
s := &Store{db: db, dirty: map[string]Entry{}, stop: make(chan struct{}), logf: logf}
|
||||
if n, err := s.prune(); err != nil {
|
||||
s.logf("gridcache: prune failed: %v", err)
|
||||
} else if n > 0 {
|
||||
@@ -118,14 +115,14 @@ func (s *Store) LoadAll() (map[string]string, error) {
|
||||
// Put queues a locator for writing. Callers pass only what CHANGED — an
|
||||
// unchanged report is the common case by a wide margin and must not reach here,
|
||||
// or the batch would carry the whole feed instead of the news in it.
|
||||
func (s *Store) Put(call, grid string) {
|
||||
func (s *Store) Put(call, grid, source string) {
|
||||
call = strings.ToUpper(strings.TrimSpace(call))
|
||||
grid = strings.TrimSpace(grid)
|
||||
if call == "" || grid == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.dirty[call] = grid
|
||||
s.dirty[call] = Entry{Grid: grid, Source: source}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -160,7 +157,7 @@ func (s *Store) Flush() error {
|
||||
return nil
|
||||
}
|
||||
batch := s.dirty
|
||||
s.dirty = map[string]string{}
|
||||
s.dirty = map[string]Entry{}
|
||||
s.mu.Unlock()
|
||||
|
||||
tx, err := s.db.Begin()
|
||||
@@ -168,8 +165,9 @@ func (s *Store) Flush() error {
|
||||
s.requeue(batch)
|
||||
return err
|
||||
}
|
||||
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at`)
|
||||
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at, source) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at,
|
||||
source = excluded.source`)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
s.requeue(batch)
|
||||
@@ -177,8 +175,8 @@ func (s *Store) Flush() error {
|
||||
}
|
||||
defer st.Close()
|
||||
now := time.Now().Unix()
|
||||
for call, grid := range batch {
|
||||
if _, err := st.Exec(call, grid, now); err != nil {
|
||||
for call, e := range batch {
|
||||
if _, err := st.Exec(call, e.Grid, now, e.Source); err != nil {
|
||||
tx.Rollback()
|
||||
s.requeue(batch)
|
||||
return err
|
||||
@@ -193,17 +191,17 @@ func (s *Store) Flush() error {
|
||||
|
||||
// requeue puts a failed batch back, without overwriting anything learnt while it
|
||||
// was in flight — the newer value is the right one.
|
||||
func (s *Store) requeue(batch map[string]string) {
|
||||
func (s *Store) requeue(batch map[string]Entry) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for call, grid := range batch {
|
||||
for call, e := range batch {
|
||||
if _, newer := s.dirty[call]; !newer {
|
||||
s.dirty[call] = grid
|
||||
s.dirty[call] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pending reports how many locators are waiting to be written (for diagnostics).
|
||||
// Pending reports how many locators are waiting to be written.
|
||||
func (s *Store) Pending() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -21,8 +21,8 @@ func open(t *testing.T) (*Store, string) {
|
||||
// of after an hour of listening.
|
||||
func TestSurvivesRestart(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("F4BPO", "JN36")
|
||||
s.Put("OH5CX", "KP30")
|
||||
s.Put("F4BPO", "JN36", SourceDecode)
|
||||
s.Put("OH5CX", "KP30", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatalf("flush: %v", err)
|
||||
}
|
||||
@@ -51,11 +51,11 @@ func TestNewestReportWins(t *testing.T) {
|
||||
s, _ := open(t)
|
||||
defer s.Close()
|
||||
|
||||
s.Put("F4BPO", "JN36")
|
||||
s.Put("F4BPO", "JN36", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Put("F4BPO", "KP30") // moved
|
||||
s.Put("F4BPO", "KP30", SourceDecode) // moved
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func TestBatching(t *testing.T) {
|
||||
defer s.Close()
|
||||
|
||||
for _, c := range []string{"A1AA", "B2BB", "C3CC"} {
|
||||
s.Put(c, "JN36")
|
||||
s.Put(c, "JN36", SourceDecode)
|
||||
}
|
||||
if n := s.Pending(); n != 3 {
|
||||
t.Errorf("pending = %d, want 3 queued and unwritten", n)
|
||||
@@ -106,8 +106,8 @@ func TestBatching(t *testing.T) {
|
||||
// square is the one way this cache can be actively wrong rather than empty.
|
||||
func TestPruneOnOpen(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("FRESH", "JN36")
|
||||
s.Put("STALE", "IO91")
|
||||
s.Put("FRESH", "JN36", SourceDecode)
|
||||
s.Put("STALE", "IO91", SourceDecode)
|
||||
if err := s.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func TestPruneOnOpen(t *testing.T) {
|
||||
// trade.
|
||||
func TestCloseFlushes(t *testing.T) {
|
||||
s, path := open(t)
|
||||
s.Put("LATE", "JN36")
|
||||
s.Put("LATE", "JN36", SourceDecode)
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
+44
-5
@@ -81,7 +81,16 @@ type Config struct {
|
||||
// OnSpot receives every accepted decode. Called from the MQTT goroutine, so
|
||||
// it must not block: the broker's buffer is what pays for it if it does.
|
||||
OnSpot func(Spot)
|
||||
Logf func(string, ...any)
|
||||
// OnGrid receives the transmitter of EVERY message, before any geographic
|
||||
// filtering, for the callsign-to-locator store. Same goroutine as OnSpot and
|
||||
// the same rule: do not block.
|
||||
OnGrid func(call, grid string)
|
||||
// RxGrids filters at the BROKER: only reports collected by a receiver in one
|
||||
// of these squares are sent at all. Empty keeps the old behaviour, which was
|
||||
// to receive the world and discard it here — measured at 83 messages a second
|
||||
// for the four opening bands, of which about one in a hundred survived.
|
||||
RxGrids []string
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
|
||||
// Watcher owns the MQTT connection and its subscriptions.
|
||||
@@ -115,6 +124,32 @@ func New(cfg Config) *Watcher {
|
||||
return &Watcher{cfg: cfg}
|
||||
}
|
||||
|
||||
// topics builds the subscription list.
|
||||
//
|
||||
// The v2 topic is
|
||||
//
|
||||
// pskr/filter/v2/<band>/<mode>/<tx call>/<rx call>/<tx grid>/<rx grid>/<tx dxcc>/<rx dxcc>
|
||||
//
|
||||
// so the receiver's square is a level the broker can filter on, and a band of
|
||||
// "+" means every band. Filtering by RECEIVER square rather than by receiver
|
||||
// DXCC is deliberate: measured on 20 m, one country ranged from 1.2 messages a
|
||||
// second (OH) to 72.5 (K), because a DXCC can be a continent. By square the
|
||||
// same measurement is 0.2 to 1.2 — the load follows distance, which is what the
|
||||
// feed is actually about, and it is the same for every operator.
|
||||
func (w *Watcher) topics() []string {
|
||||
out := []string{}
|
||||
for _, b := range w.cfg.Bands {
|
||||
if len(w.cfg.RxGrids) == 0 {
|
||||
out = append(out, "pskr/filter/v2/"+b+"/#")
|
||||
continue
|
||||
}
|
||||
for _, g := range w.cfg.RxGrids {
|
||||
out = append(out, "pskr/filter/v2/"+b+"/+/+/+/+/"+strings.ToUpper(g)+"/+/+")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Start connects and subscribes. Safe to call when already running.
|
||||
func (w *Watcher) Start() error {
|
||||
w.mu.Lock()
|
||||
@@ -143,10 +178,7 @@ func (w *Watcher) Start() error {
|
||||
|
||||
opts.OnConnect = func(c mqtt.Client) {
|
||||
w.cfg.Logf("pskr: connected to %s", w.cfg.Broker)
|
||||
for _, b := range w.cfg.Bands {
|
||||
// Every mode, every pair of stations, on this band. That firehose IS
|
||||
// the point: the detector's job is to find the shape in it.
|
||||
topic := "pskr/filter/v2/" + b + "/#"
|
||||
for _, topic := range w.topics() {
|
||||
if tok := c.Subscribe(topic, 0, w.handle); tok.Wait() && tok.Error() != nil {
|
||||
w.cfg.Logf("pskr: subscribe %s failed: %v", topic, tok.Error())
|
||||
continue
|
||||
@@ -206,6 +238,13 @@ func (w *Watcher) handle(_ mqtt.Client, m mqtt.Message) {
|
||||
return
|
||||
}
|
||||
|
||||
// The locator store takes every transmitter, before any of the geography
|
||||
// below. What it wants is "which square is this callsign in", and that is
|
||||
// true whoever happened to hear the report.
|
||||
if w.cfg.OnGrid != nil {
|
||||
w.cfg.OnGrid(call, grid[:4])
|
||||
}
|
||||
|
||||
// THE RECEIVER HAS TO BE NEAR THE OPERATOR. This is the whole difference
|
||||
// between a useful feed and a world map.
|
||||
//
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package pskr
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The receiver square is a level the BROKER can filter on, which is the whole
|
||||
// point: measured on the live feed, the four opening bands unfiltered are 83
|
||||
// messages a second of which about one in a hundred survives the NearKm test.
|
||||
// One ring of squares is under two a second, and the same for every operator —
|
||||
// where filtering by DXCC ranged from 1.2 (OH) to 72.5 (K) on one band.
|
||||
func TestTopicsFilterOnTheReceiverSquare(t *testing.T) {
|
||||
w := New(Config{Bands: []string{"6m"}, RxGrids: []string{"jn36", "JN37"}})
|
||||
got := w.topics()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want one subscription per band × square, got %v", got)
|
||||
}
|
||||
// Level order: band/mode/txcall/rxcall/txgrid/RXGRID/txdxcc/rxdxcc
|
||||
want := "pskr/filter/v2/6m/+/+/+/+/JN36/+/+"
|
||||
if got[0] != want {
|
||||
t.Errorf("topic = %q, want %q", got[0], want)
|
||||
}
|
||||
if !strings.Contains(got[1], "/JN37/") {
|
||||
t.Errorf("square not upper-cased into the topic: %q", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
// With no squares the old behaviour stands: receive the band and decide here.
|
||||
func TestTopicsWithoutSquares(t *testing.T) {
|
||||
w := New(Config{Bands: []string{"10m", "2m"}})
|
||||
got := w.topics()
|
||||
if len(got) != 2 || got[0] != "pskr/filter/v2/10m/#" {
|
||||
t.Errorf("unfiltered topics = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Grid chasing wants every band. "+" is the MQTT single-level wildcard, so one
|
||||
// subscription per square covers the lot instead of one per band per square.
|
||||
func TestTopicsAllBands(t *testing.T) {
|
||||
w := New(Config{Bands: []string{"+"}, RxGrids: []string{"JN36"}})
|
||||
got := w.topics()
|
||||
if len(got) != 1 || got[0] != "pskr/filter/v2/+/+/+/+/+/JN36/+/+" {
|
||||
t.Errorf("all-band topic = %v", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user