feat(cluster): persist learnt locators behind a "Chase new grids" option
The callsign->grid map died with the process. Every restart began with an empty Locator column that took an hour of listening to refill, and everything learnt the day before was thrown away. internal/gridcache is its own SQLite file in the data directory, not a table in the settings database: that one sits wherever the operator chose to put it, often a synchronised folder, and a store that rewrites itself every minute has no business there. Deleting the file costs a few days of listening and nothing else. Callsign is the primary key and the newest report wins — operators move, go portable, go on expedition, and a stale locator is worse than none for grid chasing because it reads as a square already worked. Entries not seen in two years are pruned at open: that is the one way this cache can be actively wrong rather than merely empty, and it is what bounds a store that would otherwise only grow. Only CHANGES are queued. The feeds repeat themselves, so writing every report would put the whole stream in the batch instead of the news in it. Batches flush on a timer and on shutdown — a restart is exactly when the cache is worth the most. Rotation is switched off while the store is attached: the cap would discard callsigns the database still holds and the lookup would then miss what we know. Age in the store is the bound instead.
This commit is contained in:
@@ -41,6 +41,7 @@ import (
|
|||||||
"hamlog/internal/email"
|
"hamlog/internal/email"
|
||||||
"hamlog/internal/extsvc"
|
"hamlog/internal/extsvc"
|
||||||
"hamlog/internal/geo"
|
"hamlog/internal/geo"
|
||||||
|
"hamlog/internal/gridcache"
|
||||||
"hamlog/internal/integrations/udp"
|
"hamlog/internal/integrations/udp"
|
||||||
"hamlog/internal/lookup"
|
"hamlog/internal/lookup"
|
||||||
"hamlog/internal/lotwusers"
|
"hamlog/internal/lotwusers"
|
||||||
@@ -235,6 +236,7 @@ const (
|
|||||||
keyUltrabeamStep = "ultrabeam.step_khz" // re-tune hysteresis: 25 | 50 | 100 kHz
|
keyUltrabeamStep = "ultrabeam.step_khz" // re-tune hysteresis: 25 | 50 | 100 kHz
|
||||||
keyMotorTrackMode = "motor.track_mode" // "always" | "step" | "band"
|
keyMotorTrackMode = "motor.track_mode" // "always" | "step" | "band"
|
||||||
keyMotorBandFreqs = "motor.band_freqs" // per-band tune frequency: "40m=7100,20m=14150"
|
keyMotorBandFreqs = "motor.band_freqs" // per-band tune frequency: "40m=7100,20m=14150"
|
||||||
|
keyChaseNewGrids = "cluster.chase_grids" // "1" → persist learnt locators across restarts
|
||||||
keyMotorType = "ultrabeam.type" // "ultrabeam" | "steppir" (default ultrabeam)
|
keyMotorType = "ultrabeam.type" // "ultrabeam" | "steppir" (default ultrabeam)
|
||||||
keyMotorTransport = "ultrabeam.transport" // "tcp" | "serial" (default tcp)
|
keyMotorTransport = "ultrabeam.transport" // "tcp" | "serial" (default tcp)
|
||||||
keyMotorCOM = "ultrabeam.com" // serial device name (COM3, /dev/ttyUSB0)
|
keyMotorCOM = "ultrabeam.com" // serial device name (COM3, /dev/ttyUSB0)
|
||||||
@@ -622,6 +624,12 @@ type App struct {
|
|||||||
decodeGrids map[string]string // current generation, written to
|
decodeGrids map[string]string // current generation, written to
|
||||||
decodeGridsOld map[string]string // previous generation, still readable
|
decodeGridsOld map[string]string // previous generation, still readable
|
||||||
decodeGridsMu sync.RWMutex
|
decodeGridsMu sync.RWMutex
|
||||||
|
// gridStore persists the map across restarts when grid chasing is on. With
|
||||||
|
// it, rotation is switched OFF: rotating would drop callsigns the database
|
||||||
|
// still holds, and a lookup would then miss something we know. The store
|
||||||
|
// bounds itself by age instead, so memory follows how many distinct stations
|
||||||
|
// have actually been heard in two years rather than a made-up ceiling.
|
||||||
|
gridStore *gridcache.Store
|
||||||
// pskr is the PSK Reporter MQTT feed, up only while the opening watch is on.
|
// pskr is the PSK Reporter MQTT feed, up only while the opening watch is on.
|
||||||
// It is the source that makes VHF detection work at all: the cluster and RBN
|
// It is the source that makes VHF detection work at all: the cluster and RBN
|
||||||
// carry a handful of 6 m spots where PSK Reporter carries hundreds.
|
// carry a handful of 6 m spots where PSK Reporter carries hundreds.
|
||||||
@@ -1387,6 +1395,10 @@ func (a *App) startup(ctx context.Context) {
|
|||||||
// PSK Reporter, when the opening watch is on. After the operator's grid is
|
// PSK Reporter, when the opening watch is on. After the operator's grid is
|
||||||
// known: without it there is no distance to measure and the feed stays down.
|
// known: without it there is no distance to measure and the feed stays down.
|
||||||
a.startBandOpenFeed()
|
a.startBandOpenFeed()
|
||||||
|
// Locator store. After settings are scoped, so the option is read from the
|
||||||
|
// right profile, and before the cluster starts serving statuses so the first
|
||||||
|
// spots already carry their locators.
|
||||||
|
a.startGridCache()
|
||||||
// One-time tidy-up of a field nothing used to record. Background, once.
|
// One-time tidy-up of a field nothing used to record. Background, once.
|
||||||
a.backfillDistancesOnce()
|
a.backfillDistancesOnce()
|
||||||
|
|
||||||
@@ -1634,6 +1646,14 @@ func (a *App) shutdown(ctx context.Context) {
|
|||||||
if a.qsoRec != nil {
|
if a.qsoRec != nil {
|
||||||
a.qsoRec.Stop()
|
a.qsoRec.Stop()
|
||||||
}
|
}
|
||||||
|
// Before the databases: Close flushes what the last minute learnt, and a
|
||||||
|
// restart is exactly when the grid cache is worth the most.
|
||||||
|
if a.gridStore != nil {
|
||||||
|
if err := a.gridStore.Close(); err != nil {
|
||||||
|
applog.Printf("gridcache: close: %v", err)
|
||||||
|
}
|
||||||
|
a.gridStore = nil
|
||||||
|
}
|
||||||
if a.logDb != nil && a.logDb != a.db {
|
if a.logDb != nil && a.logDb != a.db {
|
||||||
_ = a.logDb.Close() // shared MySQL logbook (separate from the local config DB)
|
_ = a.logDb.Close() // shared MySQL logbook (separate from the local config DB)
|
||||||
}
|
}
|
||||||
@@ -17023,6 +17043,65 @@ func (a *App) clusterStatusMaps() *clusterStatusCache {
|
|||||||
// decodes and the ceiling was never reached anyway.
|
// decodes and the ceiling was never reached anyway.
|
||||||
const decodeGridsCap = 100000
|
const decodeGridsCap = 100000
|
||||||
|
|
||||||
|
// GetChaseNewGrids reports whether learnt locators are kept across restarts.
|
||||||
|
func (a *App) GetChaseNewGrids() bool { return a.settingOr(keyChaseNewGrids, "") == "1" }
|
||||||
|
|
||||||
|
// SetChaseNewGrids turns grid chasing on or off and applies it immediately.
|
||||||
|
func (a *App) SetChaseNewGrids(on bool) error {
|
||||||
|
a.setSetting(keyChaseNewGrids, boolStr(on))
|
||||||
|
a.startGridCache()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// startGridCache brings the locator store up or down to match the setting.
|
||||||
|
//
|
||||||
|
// Switching it ON seeds the in-memory map from the database, which is the whole
|
||||||
|
// point: the cluster's locator column is populated in the first second instead
|
||||||
|
// of after an hour of listening. Switching it OFF closes the file and leaves the
|
||||||
|
// map alone — what has already been learnt this session stays usable, it simply
|
||||||
|
// stops being remembered. Nothing is deleted: turning the option back on picks
|
||||||
|
// up where it left off.
|
||||||
|
func (a *App) startGridCache() {
|
||||||
|
if a.gridStore != nil {
|
||||||
|
if err := a.gridStore.Close(); err != nil {
|
||||||
|
applog.Printf("gridcache: close: %v", err)
|
||||||
|
}
|
||||||
|
a.gridStore = nil
|
||||||
|
}
|
||||||
|
if !a.GetChaseNewGrids() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := filepath.Join(a.dataDir, "grids.db")
|
||||||
|
st, err := gridcache.Open(path, applog.Printf)
|
||||||
|
if err != nil {
|
||||||
|
applog.Printf("gridcache: disabled — %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seed, err := st.LoadAll()
|
||||||
|
if err != nil {
|
||||||
|
applog.Printf("gridcache: cannot read %s (%v) — starting empty", path, err)
|
||||||
|
seed = map[string]string{}
|
||||||
|
}
|
||||||
|
a.decodeGridsMu.Lock()
|
||||||
|
if a.decodeGrids == nil {
|
||||||
|
a.decodeGrids = make(map[string]string, len(seed)+512)
|
||||||
|
}
|
||||||
|
// Seed UNDER what this session already heard: a locator decoded a minute ago
|
||||||
|
// is newer than one stored days back, and the newest report is the one that
|
||||||
|
// counts when a station has moved.
|
||||||
|
for call, grid := range seed {
|
||||||
|
if _, live := a.decodeGrids[call]; !live {
|
||||||
|
a.decodeGrids[call] = grid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
n := len(a.decodeGrids)
|
||||||
|
a.decodeGridsMu.Unlock()
|
||||||
|
|
||||||
|
a.gridStore = st
|
||||||
|
st.Start(a.ctx)
|
||||||
|
applog.Printf("gridcache: grid chasing on — %d locators loaded from %s (%d known in total)", len(seed), path, n)
|
||||||
|
}
|
||||||
|
|
||||||
// rememberDecodeGrid records the grid a station announced.
|
// rememberDecodeGrid records the grid a station announced.
|
||||||
//
|
//
|
||||||
// Rotation, not eviction: when the current generation fills it becomes the
|
// Rotation, not eviction: when the current generation fills it becomes the
|
||||||
@@ -17031,19 +17110,38 @@ const decodeGridsCap = 100000
|
|||||||
// because this runs once per decode.
|
// because this runs once per decode.
|
||||||
func (a *App) rememberDecodeGrid(call, grid string) {
|
func (a *App) rememberDecodeGrid(call, grid string) {
|
||||||
call = strings.ToUpper(strings.TrimSpace(call))
|
call = strings.ToUpper(strings.TrimSpace(call))
|
||||||
|
grid = strings.TrimSpace(grid)
|
||||||
if call == "" || grid == "" {
|
if call == "" || grid == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
store := a.gridStore
|
||||||
|
|
||||||
a.decodeGridsMu.Lock()
|
a.decodeGridsMu.Lock()
|
||||||
if a.decodeGrids == nil {
|
if a.decodeGrids == nil {
|
||||||
a.decodeGrids = make(map[string]string, 512)
|
a.decodeGrids = make(map[string]string, 512)
|
||||||
}
|
}
|
||||||
if len(a.decodeGrids) >= decodeGridsCap {
|
// Rotate only while nothing is persisting the map. With a store the cap would
|
||||||
|
// discard callsigns the database still holds, so the lookup would miss what
|
||||||
|
// we know; age in the store is the bound instead.
|
||||||
|
if store == nil && len(a.decodeGrids) >= decodeGridsCap {
|
||||||
a.decodeGridsOld = a.decodeGrids
|
a.decodeGridsOld = a.decodeGrids
|
||||||
a.decodeGrids = make(map[string]string, 512)
|
a.decodeGrids = make(map[string]string, 512)
|
||||||
}
|
}
|
||||||
|
// Only a CHANGE is worth writing. The feeds repeat themselves — the same
|
||||||
|
// station is reported by dozens of receivers a minute — so queueing every
|
||||||
|
// report would put the whole stream in the batch instead of the news in it.
|
||||||
|
changed := a.decodeGrids[call] != grid
|
||||||
|
if changed && a.decodeGridsOld != nil && a.decodeGridsOld[call] == grid {
|
||||||
|
// Known already, just in the older generation: promote it without calling
|
||||||
|
// it news.
|
||||||
|
changed = false
|
||||||
|
}
|
||||||
a.decodeGrids[call] = grid
|
a.decodeGrids[call] = grid
|
||||||
a.decodeGridsMu.Unlock()
|
a.decodeGridsMu.Unlock()
|
||||||
|
|
||||||
|
if changed && store != nil {
|
||||||
|
store.Put(call, grid)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookupDecodeGrid returns the last grid heard for a callsign, "" if unknown.
|
// lookupDecodeGrid returns the last grid heard for a callsign, "" if unknown.
|
||||||
|
|||||||
+4
-2
@@ -3,10 +3,12 @@
|
|||||||
"version": "0.24.8",
|
"version": "0.24.8",
|
||||||
"date": "",
|
"date": "",
|
||||||
"en": [
|
"en": [
|
||||||
"Cluster: the grid cache now holds 100,000 callsigns and rotates instead of emptying itself, so locators stop vanishing from the list."
|
"Cluster: the grid cache now holds 100,000 callsigns and rotates instead of emptying itself, so locators stop vanishing from the list.",
|
||||||
|
"New option \"Chase new grids\": locators learnt from decodes are kept in their own database, so the cluster shows them from the first second."
|
||||||
],
|
],
|
||||||
"fr": [
|
"fr": [
|
||||||
"Cluster : le cache de locators garde 100 000 indicatifs et tourne au lieu de se vider, les locators ne disparaissent donc plus de la liste."
|
"Cluster : le cache de locators garde 100 000 indicatifs et tourne au lieu de se vider, les locators ne disparaissent donc plus de la liste.",
|
||||||
|
"Nouvelle option « Chasse aux nouveaux carrés » : les locators appris des décodes sont conservés dans leur propre base, le cluster les affiche donc dès la première seconde."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2,8 +2,11 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"hamlog/internal/gridcache"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The grid cache used to drop EVERYTHING at its ceiling. That was survivable
|
// The grid cache used to drop EVERYTHING at its ceiling. That was survivable
|
||||||
@@ -95,3 +98,63 @@ func TestDecodeGridConcurrentAccess(t *testing.T) {
|
|||||||
}()
|
}()
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only a CHANGE may reach the write batch.
|
||||||
|
//
|
||||||
|
// The feeds repeat themselves — the same station is reported by dozens of
|
||||||
|
// receivers a minute — so queueing every report would put the whole stream in
|
||||||
|
// the batch instead of the news in it, and turn a cache into a write amplifier.
|
||||||
|
func TestOnlyChangesAreQueued(t *testing.T) {
|
||||||
|
st, err := gridcache.Open(filepath.Join(t.TempDir(), "grids.db"), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
a := &App{gridStore: st}
|
||||||
|
|
||||||
|
a.rememberDecodeGrid("F4BPO", "JN36")
|
||||||
|
if n := st.Pending(); n != 1 {
|
||||||
|
t.Fatalf("a new locator queued %d writes, want 1", n)
|
||||||
|
}
|
||||||
|
if err := st.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same report, a hundred times over, is not news.
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
a.rememberDecodeGrid("F4BPO", "JN36")
|
||||||
|
}
|
||||||
|
if n := st.Pending(); n != 0 {
|
||||||
|
t.Errorf("unchanged reports queued %d writes — the batch would carry the whole feed", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A station that moved is.
|
||||||
|
a.rememberDecodeGrid("F4BPO", "KP30")
|
||||||
|
if n := st.Pending(); n != 1 {
|
||||||
|
t.Errorf("a changed locator queued %d writes, want 1", n)
|
||||||
|
}
|
||||||
|
if got := a.lookupDecodeGrid("F4BPO"); got != "KP30" {
|
||||||
|
t.Errorf("the map kept the old locator: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rotation must be OFF while a store is attached: it would discard callsigns the
|
||||||
|
// database still holds, and the lookup would then miss something we know.
|
||||||
|
func TestNoRotationWhilePersisting(t *testing.T) {
|
||||||
|
st, err := gridcache.Open(filepath.Join(t.TempDir(), "grids.db"), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
a := &App{gridStore: st}
|
||||||
|
|
||||||
|
for i := 0; i < decodeGridsCap+10; i++ {
|
||||||
|
a.rememberDecodeGrid(fmt.Sprintf("CALL%06d", i), "JN36")
|
||||||
|
}
|
||||||
|
if a.decodeGridsOld != nil {
|
||||||
|
t.Error("rotated while a store was attached — locators the database holds would go missing")
|
||||||
|
}
|
||||||
|
if got := a.lookupDecodeGrid("CALL000000"); got != "JN36" {
|
||||||
|
t.Errorf("the first entry was dropped: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import {
|
|||||||
GetADIFMonitor, SaveADIFMonitor, PickADIFMonitorFile,
|
GetADIFMonitor, SaveADIFMonitor, PickADIFMonitorFile,
|
||||||
GetRelayAuto, SaveRelayAuto, GetStationDevices,
|
GetRelayAuto, SaveRelayAuto, GetStationDevices,
|
||||||
GetAwardDefs, GetTrackedAwards, SaveTrackedAwards,
|
GetAwardDefs, GetTrackedAwards, SaveTrackedAwards,
|
||||||
GetBandOpenSettings, SaveBandOpenSettings, GetPSKReporterStatus,
|
GetBandOpenSettings, SaveBandOpenSettings, GetPSKReporterStatus, GetChaseNewGrids, SetChaseNewGrids,
|
||||||
} from '../../wailsjs/go/main/App';
|
} from '../../wailsjs/go/main/App';
|
||||||
import type { profile as profileModels } from '../../wailsjs/go/models';
|
import type { profile as profileModels } from '../../wailsjs/go/models';
|
||||||
import type { LookupSettingsForm, StationSettingsForm, ListsSettingsForm, ModePresetForm } from '@/types';
|
import type { LookupSettingsForm, StationSettingsForm, ListsSettingsForm, ModePresetForm } from '@/types';
|
||||||
@@ -1552,6 +1552,7 @@ export function SettingsModal({ onClose, onSaved, initialSection, onMainPaneChan
|
|||||||
// has side effects there — adding the RBN nodes, bringing the PSK Reporter
|
// has side effects there — adding the RBN nodes, bringing the PSK Reporter
|
||||||
// feed up or down — so the write has to go where those live.
|
// feed up or down — so the write has to go where those live.
|
||||||
const [bandOpen, setBandOpen] = useState<any>({ enabled: false, bands: [], available: [] });
|
const [bandOpen, setBandOpen] = useState<any>({ enabled: false, bands: [], available: [] });
|
||||||
|
const [chaseGrids, setChaseGrids] = useState(false);
|
||||||
const [pskrStatus, setPskrStatus] = useState<any>(null);
|
const [pskrStatus, setPskrStatus] = useState<any>(null);
|
||||||
const saveBandOpen = async (next: any) => {
|
const saveBandOpen = async (next: any) => {
|
||||||
setBandOpen(next);
|
setBandOpen(next);
|
||||||
@@ -1560,6 +1561,7 @@ export function SettingsModal({ onClose, onSaved, initialSection, onMainPaneChan
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
(async () => {
|
(async () => {
|
||||||
try { setBandOpen(await GetBandOpenSettings()); } catch { /* defaults stand */ }
|
try { setBandOpen(await GetBandOpenSettings()); } catch { /* defaults stand */ }
|
||||||
|
try { setChaseGrids(await GetChaseNewGrids()); } catch { /* defaults stand */ }
|
||||||
})();
|
})();
|
||||||
// Poll the feed while the panel is open: a live count is the only thing that
|
// Poll the feed while the panel is open: a live count is the only thing that
|
||||||
// distinguishes "connected" from "connected and receiving nothing".
|
// distinguishes "connected" from "connected and receiving nothing".
|
||||||
@@ -4227,6 +4229,17 @@ export function SettingsModal({ onClose, onSaved, initialSection, onMainPaneChan
|
|||||||
things set up once. A preferences dialog you reopen every ten minutes
|
things set up once. A preferences dialog you reopen every ten minutes
|
||||||
is a filter in the wrong place. */}
|
is a filter in the wrong place. */}
|
||||||
|
|
||||||
|
{/* Grid chasing. Here rather than in the filter panel because it is set
|
||||||
|
up once: it decides whether locators learnt from decodes are KEPT
|
||||||
|
across restarts, not what the list shows right now. */}
|
||||||
|
<div className="border-t border-border/60 pt-3 space-y-2">
|
||||||
|
<label className="flex items-start gap-2 text-sm cursor-pointer">
|
||||||
|
<Checkbox checked={chaseGrids} className="mt-0.5"
|
||||||
|
onCheckedChange={(c) => { setChaseGrids(!!c); SetChaseNewGrids(!!c).catch(() => {}); }} />
|
||||||
|
<span>{t('clu.chaseGrids')} <span className="text-xs text-muted-foreground">{t('clu.chaseGridsHint')}</span></span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* Band-opening watch. It lives HERE, with the cluster nodes, because
|
{/* Band-opening watch. It lives HERE, with the cluster nodes, because
|
||||||
switching it on adds two of them — the operator should see that
|
switching it on adds two of them — the operator should see that
|
||||||
happen where it happens rather than find nodes they did not add. */}
|
happen where it happens rather than find nodes they did not add. */}
|
||||||
|
|||||||
@@ -270,7 +270,7 @@ const en: Dict = {
|
|||||||
'clu.muteWorkedHint': '(they stay in the list, just quiet — leaves the colour for what is left to do)',
|
'clu.muteWorkedHint': '(they stay in the list, just quiet — leaves the colour for what is left to do)',
|
||||||
'clu.slotHighlight': 'Colour the stations not worked on this band and mode',
|
'clu.slotHighlight': 'Colour the stations not worked on this band and mode',
|
||||||
'clu.slotHighlightHint': '(by callsign, whatever the entity status says)',
|
'clu.slotHighlightHint': '(by callsign, whatever the entity status says)',
|
||||||
'bo.open': 'open', 'bo.liveTip': '{band} is open — {n} stations, ~{km} km, {sector}{season}. Click for the band map.', 'bo.enable': 'Watch for band openings', 'bo.enableHint': '(10, 12, 6, 4 and 2 m. Switching this on adds the two RBN nodes and subscribes to the PSK Reporter feed — the detection needs far more ears than a cluster can give it.)', 'bo.feedUp': 'PSK Reporter feed up — {n} decodes seen', 'bo.feedDown': 'PSK Reporter feed down — needs your station grid, and a moment to connect', 'clu.workedSameSlot': 'Already worked only on the same slot',
|
'bo.open': 'open', 'bo.liveTip': '{band} is open — {n} stations, ~{km} km, {sector}{season}. Click for the band map.', 'bo.enable': 'Watch for band openings', 'bo.enableHint': '(10, 12, 6, 4 and 2 m. Switching this on adds the two RBN nodes and subscribes to the PSK Reporter feed — the detection needs far more ears than a cluster can give it.)', 'bo.feedUp': 'PSK Reporter feed up — {n} decodes seen', 'bo.feedDown': 'PSK Reporter feed down — needs your station grid, and a moment to connect', 'clu.chaseGrids': 'Chase new grids', 'clu.chaseGridsHint': '(keeps the locators learnt from WSJT-X decodes in their own database, so the cluster shows them from the first second instead of after an hour of listening)', 'clu.workedSameSlot': 'Already worked only on the same slot',
|
||||||
'clu.workedSameSlotHint': '— a spot shows "worked" only if you worked that call on the SAME band and mode, not just anywhere. Combines with digital-mode grouping (Settings → General): with it on, a call worked on 20m FT8 also counts as worked for a 20m FT4 spot; with it off, FT8 and FT4 are separate slots.',
|
'clu.workedSameSlotHint': '— a spot shows "worked" only if you worked that call on the SAME band and mode, not just anywhere. Combines with digital-mode grouping (Settings → General): with it on, a call worked on 20m FT8 also counts as worked for a 20m FT4 spot; with it off, FT8 and FT4 are separate slots.',
|
||||||
// Backup panel
|
// Backup panel
|
||||||
'bk.hintMysql': 'On close (once/day) OpsLog snapshots the local SQLite (config) AND exports the shared MySQL log to ADIF — opslog-log-<date>.adi — so your contacts are protected even though they live on the server. Rotation keeps the last N of each.',
|
'bk.hintMysql': 'On close (once/day) OpsLog snapshots the local SQLite (config) AND exports the shared MySQL log to ADIF — opslog-log-<date>.adi — so your contacts are protected even though they live on the server. Rotation keeps the last N of each.',
|
||||||
|
|||||||
Vendored
+4
@@ -404,6 +404,8 @@ export function GetCatalogCodes():Promise<Array<string>>;
|
|||||||
|
|
||||||
export function GetChangelog():Promise<Array<main.ChangelogEntry>>;
|
export function GetChangelog():Promise<Array<main.ChangelogEntry>>;
|
||||||
|
|
||||||
|
export function GetChaseNewGrids():Promise<boolean>;
|
||||||
|
|
||||||
export function GetChatHistory(arg1:number):Promise<Array<main.ChatMessage>>;
|
export function GetChatHistory(arg1:number):Promise<Array<main.ChatMessage>>;
|
||||||
|
|
||||||
export function GetClublogCtyInfo():Promise<main.ClublogCtyInfo>;
|
export function GetClublogCtyInfo():Promise<main.ClublogCtyInfo>;
|
||||||
@@ -970,6 +972,8 @@ export function SetCIVTrace(arg1:boolean):Promise<void>;
|
|||||||
|
|
||||||
export function SetCWDecoderPitch(arg1:number):Promise<void>;
|
export function SetCWDecoderPitch(arg1:number):Promise<void>;
|
||||||
|
|
||||||
|
export function SetChaseNewGrids(arg1:boolean):Promise<void>;
|
||||||
|
|
||||||
export function SetClublogCtyEnabled(arg1:boolean):Promise<void>;
|
export function SetClublogCtyEnabled(arg1:boolean):Promise<void>;
|
||||||
|
|
||||||
export function SetClublogMostWantedEnabled(arg1:boolean):Promise<void>;
|
export function SetClublogMostWantedEnabled(arg1:boolean):Promise<void>;
|
||||||
|
|||||||
@@ -750,6 +750,10 @@ export function GetChangelog() {
|
|||||||
return window['go']['main']['App']['GetChangelog']();
|
return window['go']['main']['App']['GetChangelog']();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function GetChaseNewGrids() {
|
||||||
|
return window['go']['main']['App']['GetChaseNewGrids']();
|
||||||
|
}
|
||||||
|
|
||||||
export function GetChatHistory(arg1) {
|
export function GetChatHistory(arg1) {
|
||||||
return window['go']['main']['App']['GetChatHistory'](arg1);
|
return window['go']['main']['App']['GetChatHistory'](arg1);
|
||||||
}
|
}
|
||||||
@@ -1882,6 +1886,10 @@ export function SetCWDecoderPitch(arg1) {
|
|||||||
return window['go']['main']['App']['SetCWDecoderPitch'](arg1);
|
return window['go']['main']['App']['SetCWDecoderPitch'](arg1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function SetChaseNewGrids(arg1) {
|
||||||
|
return window['go']['main']['App']['SetChaseNewGrids'](arg1);
|
||||||
|
}
|
||||||
|
|
||||||
export function SetClublogCtyEnabled(arg1) {
|
export function SetClublogCtyEnabled(arg1) {
|
||||||
return window['go']['main']['App']['SetClublogCtyEnabled'](arg1);
|
return window['go']['main']['App']['SetClublogCtyEnabled'](arg1);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,224 @@
|
|||||||
|
// Package gridcache is the long-term callsign→grid store behind grid chasing.
|
||||||
|
//
|
||||||
|
// A DX-cluster line never carries the DX's locator, so a spot can only show one
|
||||||
|
// if OpsLog learnt it elsewhere: a CQ decoded over the WSJT-X UDP link, or a
|
||||||
|
// PSK Reporter report. Learning it is easy; the problem is that the knowledge
|
||||||
|
// died with the process. Every restart began with an empty column that took an
|
||||||
|
// hour of listening to fill, and everything learnt yesterday was thrown away.
|
||||||
|
//
|
||||||
|
// So the map is persisted. Three things follow from what it is:
|
||||||
|
//
|
||||||
|
// - It is a CACHE, not user data. Deleting the file costs a few days of
|
||||||
|
// listening and nothing else, which is why it lives in its own file rather
|
||||||
|
// than in the settings database — that one sits wherever the operator chose
|
||||||
|
// to put it, often a synchronised folder, and a store that rewrites itself
|
||||||
|
// every minute has no business there.
|
||||||
|
//
|
||||||
|
// - A callsign has ONE grid. The key is unique and the newest report wins:
|
||||||
|
// operators move, go portable, go on expedition. A stale locator is worse
|
||||||
|
// than none for grid chasing, because it reads as a square already worked.
|
||||||
|
//
|
||||||
|
// - Writes are batched. The feeds repeat themselves — the same station is
|
||||||
|
// reported by dozens of receivers a minute — so the store accumulates
|
||||||
|
// changes in memory and flushes them on a timer. Nothing on the ingest path
|
||||||
|
// touches the disk.
|
||||||
|
package gridcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Retention bounds a store that would otherwise only ever grow. Two years is
|
||||||
|
// chosen to be far longer than any propagation interest and short enough that a
|
||||||
|
// reassigned callsign eventually stops carrying its previous holder's square —
|
||||||
|
// the one way this cache can be actively wrong rather than merely empty.
|
||||||
|
const Retention = 2 * 365 * 24 * time.Hour
|
||||||
|
|
||||||
|
// FlushEvery is the batch interval. Long enough that a burst of reports costs
|
||||||
|
// one transaction, short enough that a crash loses a minute of learning.
|
||||||
|
const FlushEvery = 60 * time.Second
|
||||||
|
|
||||||
|
type Store struct {
|
||||||
|
db *sql.DB
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
dirty map[string]string // call → grid, waiting to be written
|
||||||
|
|
||||||
|
stop chan struct{}
|
||||||
|
stopOnce sync.Once
|
||||||
|
wg sync.WaitGroup
|
||||||
|
|
||||||
|
logf func(string, ...any)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open creates or opens the store at path and prunes what has aged out.
|
||||||
|
func Open(path string, logf func(string, ...any)) (*Store, error) {
|
||||||
|
if logf == nil {
|
||||||
|
logf = func(string, ...any) {}
|
||||||
|
}
|
||||||
|
// WAL so a flush never blocks a read, and a busy timeout because the flush
|
||||||
|
// goroutine and the startup load can overlap on a slow disk.
|
||||||
|
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gridcache: open %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS grids (
|
||||||
|
call TEXT PRIMARY KEY,
|
||||||
|
grid TEXT NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL
|
||||||
|
)`); err != nil {
|
||||||
|
db.Close()
|
||||||
|
return nil, fmt.Errorf("gridcache: schema: %w", err)
|
||||||
|
}
|
||||||
|
s := &Store{db: db, dirty: map[string]string{}, stop: make(chan struct{}), logf: logf}
|
||||||
|
if n, err := s.prune(); err != nil {
|
||||||
|
s.logf("gridcache: prune failed: %v", err)
|
||||||
|
} else if n > 0 {
|
||||||
|
s.logf("gridcache: pruned %d locators not heard in %d days", n, int(Retention.Hours()/24))
|
||||||
|
}
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) prune() (int64, error) {
|
||||||
|
cut := time.Now().Add(-Retention).Unix()
|
||||||
|
res, err := s.db.Exec(`DELETE FROM grids WHERE updated_at < ?`, cut)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return res.RowsAffected()
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadAll returns every stored locator, for seeding the in-memory map at
|
||||||
|
// startup. One query and one map build — the point of the whole package is that
|
||||||
|
// nothing afterwards has to ask the database anything.
|
||||||
|
func (s *Store) LoadAll() (map[string]string, error) {
|
||||||
|
rows, err := s.db.Query(`SELECT call, grid FROM grids`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("gridcache: load: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := make(map[string]string, 4096)
|
||||||
|
for rows.Next() {
|
||||||
|
var call, grid string
|
||||||
|
if err := rows.Scan(&call, &grid); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[call] = grid
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put queues a locator for writing. Callers pass only what CHANGED — an
|
||||||
|
// unchanged report is the common case by a wide margin and must not reach here,
|
||||||
|
// or the batch would carry the whole feed instead of the news in it.
|
||||||
|
func (s *Store) Put(call, grid string) {
|
||||||
|
call = strings.ToUpper(strings.TrimSpace(call))
|
||||||
|
grid = strings.TrimSpace(grid)
|
||||||
|
if call == "" || grid == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.dirty[call] = grid
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start runs the flush loop until Close.
|
||||||
|
func (s *Store) Start(ctx context.Context) {
|
||||||
|
s.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
t := time.NewTicker(FlushEvery)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-s.stop:
|
||||||
|
return
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
s.logf("gridcache: flush failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush writes the pending batch in one transaction. Safe to call with nothing
|
||||||
|
// pending, which is most of the time on a quiet band.
|
||||||
|
func (s *Store) Flush() error {
|
||||||
|
s.mu.Lock()
|
||||||
|
if len(s.dirty) == 0 {
|
||||||
|
s.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
batch := s.dirty
|
||||||
|
s.dirty = map[string]string{}
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
tx, err := s.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
s.requeue(batch)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
st, err := tx.Prepare(`INSERT INTO grids (call, grid, updated_at) VALUES (?, ?, ?)
|
||||||
|
ON CONFLICT(call) DO UPDATE SET grid = excluded.grid, updated_at = excluded.updated_at`)
|
||||||
|
if err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
s.requeue(batch)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for call, grid := range batch {
|
||||||
|
if _, err := st.Exec(call, grid, now); err != nil {
|
||||||
|
tx.Rollback()
|
||||||
|
s.requeue(batch)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
s.requeue(batch)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// requeue puts a failed batch back, without overwriting anything learnt while it
|
||||||
|
// was in flight — the newer value is the right one.
|
||||||
|
func (s *Store) requeue(batch map[string]string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
for call, grid := range batch {
|
||||||
|
if _, newer := s.dirty[call]; !newer {
|
||||||
|
s.dirty[call] = grid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pending reports how many locators are waiting to be written (for diagnostics).
|
||||||
|
func (s *Store) Pending() int {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return len(s.dirty)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close stops the loop and writes whatever is pending. A restart is the moment
|
||||||
|
// the cache is most valuable, so losing the last minute of learning to a clean
|
||||||
|
// shutdown would be a poor trade.
|
||||||
|
func (s *Store) Close() error {
|
||||||
|
s.stopOnce.Do(func() { close(s.stop) })
|
||||||
|
s.wg.Wait()
|
||||||
|
err := s.Flush()
|
||||||
|
if cerr := s.db.Close(); err == nil {
|
||||||
|
err = cerr
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package gridcache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func open(t *testing.T) (*Store, string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "grids.db")
|
||||||
|
s, err := Open(path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
return s, path
|
||||||
|
}
|
||||||
|
|
||||||
|
// The whole reason the store exists: what was learnt is still there after a
|
||||||
|
// restart, so the cluster's locator column is full in the first second instead
|
||||||
|
// of after an hour of listening.
|
||||||
|
func TestSurvivesRestart(t *testing.T) {
|
||||||
|
s, path := open(t)
|
||||||
|
s.Put("F4BPO", "JN36")
|
||||||
|
s.Put("OH5CX", "KP30")
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Fatalf("flush: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.Close(); err != nil {
|
||||||
|
t.Fatalf("close: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
again, err := Open(path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reopen: %v", err)
|
||||||
|
}
|
||||||
|
defer again.Close()
|
||||||
|
got, err := again.LoadAll()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
if got["F4BPO"] != "JN36" || got["OH5CX"] != "KP30" {
|
||||||
|
t.Errorf("locators lost across a restart: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A callsign has ONE grid and the newest report wins. Operators move, go
|
||||||
|
// portable, go on expedition — and a stale locator is worse than none for grid
|
||||||
|
// chasing, because it reads as a square already worked.
|
||||||
|
func TestNewestReportWins(t *testing.T) {
|
||||||
|
s, _ := open(t)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
s.Put("F4BPO", "JN36")
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.Put("F4BPO", "KP30") // moved
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := s.LoadAll()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got["F4BPO"] != "KP30" {
|
||||||
|
t.Errorf("grid = %q, want the newer KP30", got["F4BPO"])
|
||||||
|
}
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Errorf("a callsign must hold one row, got %d: %v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing reaches the disk until a flush, and a flush with nothing pending is
|
||||||
|
// not an error — that is most minutes on a quiet band.
|
||||||
|
func TestBatching(t *testing.T) {
|
||||||
|
s, _ := open(t)
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
for _, c := range []string{"A1AA", "B2BB", "C3CC"} {
|
||||||
|
s.Put(c, "JN36")
|
||||||
|
}
|
||||||
|
if n := s.Pending(); n != 3 {
|
||||||
|
t.Errorf("pending = %d, want 3 queued and unwritten", n)
|
||||||
|
}
|
||||||
|
if got, _ := s.LoadAll(); len(got) != 0 {
|
||||||
|
t.Errorf("wrote before the flush: %v", got)
|
||||||
|
}
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := s.Pending(); n != 0 {
|
||||||
|
t.Errorf("pending = %d after a flush, want 0", n)
|
||||||
|
}
|
||||||
|
if got, _ := s.LoadAll(); len(got) != 3 {
|
||||||
|
t.Errorf("flush wrote %d rows, want 3", len(got))
|
||||||
|
}
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Errorf("empty flush must be a no-op, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Age is what bounds a store that would otherwise only grow. A callsign not
|
||||||
|
// heard in two years is likely reassigned, and carrying its previous holder's
|
||||||
|
// square is the one way this cache can be actively wrong rather than empty.
|
||||||
|
func TestPruneOnOpen(t *testing.T) {
|
||||||
|
s, path := open(t)
|
||||||
|
s.Put("FRESH", "JN36")
|
||||||
|
s.Put("STALE", "IO91")
|
||||||
|
if err := s.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Backdate one row past the retention window.
|
||||||
|
old := time.Now().Add(-Retention - 24*time.Hour).Unix()
|
||||||
|
if _, err := s.db.Exec(`UPDATE grids SET updated_at = ? WHERE call = 'STALE'`, old); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s.Close()
|
||||||
|
|
||||||
|
again, err := Open(path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer again.Close()
|
||||||
|
got, _ := again.LoadAll()
|
||||||
|
if _, ok := got["STALE"]; ok {
|
||||||
|
t.Error("an entry past the retention window survived — the store is unbounded")
|
||||||
|
}
|
||||||
|
if got["FRESH"] != "JN36" {
|
||||||
|
t.Error("pruning took a live entry with it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close has to write what the last minute learnt: a restart is exactly when the
|
||||||
|
// cache is worth the most, so losing it to a clean shutdown would be a poor
|
||||||
|
// trade.
|
||||||
|
func TestCloseFlushes(t *testing.T) {
|
||||||
|
s, path := open(t)
|
||||||
|
s.Put("LATE", "JN36")
|
||||||
|
if err := s.Close(); err != nil {
|
||||||
|
t.Fatalf("close: %v", err)
|
||||||
|
}
|
||||||
|
again, err := Open(path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer again.Close()
|
||||||
|
got, _ := again.LoadAll()
|
||||||
|
if got["LATE"] != "JN36" {
|
||||||
|
t.Error("what was pending at shutdown was dropped")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user