rouggy/OpsLog
0
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 48 Wiki Activity

fix: QRZ confirmation download read a third of a large logbook, then skipped the rest for good

Browse Source 
Three faults compounding, all visible in one operator's report (117k QSOs):

1. We asked for OPTION=ALL every time, so QRZ serialised the entire logbook —
   56 MB — and cut it short. The parser stopped at record 27832 with "unexpected
   EOF" and everything past that was never read. The window is now sent to the
   server (OPTION=AFTER:date); the client-side date filter stays as a backstop.

2. The last-download date was stored RIGHT AFTER the fetch, before knowing
   whether the records had been processed. So the truncated run still advanced
   the window past data it had never read — and every later run skipped it. It
   is now written at the end, and only when the ADIF parsed cleanly. A window
   that moves past unseen data is worse than one that re-reads.

3. Which is why the run in the report ended "Confirmed 0, added 0 (of 0
   returned)" despite parsing 27832 records: an earlier truncated run had
   already pushed the window to yesterday, so every record was filtered out
   client-side. The operator could not recover without clearing the key by hand.

A truncated download now says so in plain words, and says the window was not
moved.
This commit is contained in:
rouggy
2026-07-28 09:52:20 +02:00
parent 843dccf0c4
commit 0a1569dbbd
2 changed files with 33 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.go
+29 -18
View File
@@ -95,11 +95,11 @@ const (
keyListsRSTCW = "lists.rst_cw"
keyListsRSTDigital = "lists.rst_digital"
keyCATEnabled = "cat.enabled"
keyCATBackend = "cat.backend" // "omnirig" | "flex"
keyCATOmniRigNum = "cat.omnirig.rig" // 1 or 2
keyCATEnabled = "cat.enabled"
keyCATBackend = "cat.backend" // "omnirig" | "flex"
keyCATOmniRigNum = "cat.omnirig.rig" // 1 or 2
// Which VFO to believe when OmniRig names one. "" = trust the rig file.
keyCATOmniRigVFO = "cat.omnirig.vfo" // "" | "A" | "B"
keyCATOmniRigVFO = "cat.omnirig.vfo" // "" | "A" | "B"
keyCATFlexHost = "cat.flex.host" // FlexRadio IP (native backend)
keyCATFlexPort = "cat.flex.port" // FlexRadio TCP port (default 4992)
keyCATFlexSpots = "cat.flex.spots" // push cluster spots to the panadapter
@@ -329,14 +329,14 @@ type QSLDefaults struct {
// CATSettings is the user-tweakable rig-control configuration. Stored as
// individual key/value pairs to keep the settings table flat.
type CATSettings struct {
Enabled bool `json:"enabled"`
Backend string `json:"backend"` // "omnirig" | "flex" | "icom" | "icom-net" | "tci"
OmniRigNum int `json:"omnirig_rig"` // 1 or 2 (OmniRig "Rig1"/"Rig2" slot)
Enabled bool `json:"enabled"`
Backend string `json:"backend"` // "omnirig" | "flex" | "icom" | "icom-net" | "tci"
OmniRigNum int `json:"omnirig_rig"` // 1 or 2 (OmniRig "Rig1"/"Rig2" slot)
// OmniRigVFO overrides which VFO OpsLog reads: "" follows what the rig file
// reports, "A"/"B" force one. Needed because that report is only as good as
// the .ini: an IC-7610 file was seen declaring VFO B permanently while the
// operator worked on the main VFO, so OpsLog wrote to A and read B.
OmniRigVFO string `json:"omnirig_vfo"` // "" | "A" | "B"
OmniRigVFO string `json:"omnirig_vfo"` // "" | "A" | "B"
FlexHost string `json:"flex_host"` // FlexRadio IP (native backend)
FlexPort int `json:"flex_port"` // FlexRadio TCP port (default 4992)
FlexSpots bool `json:"flex_spots"` // push cluster spots to the panadapter
@@ -9562,7 +9562,16 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
} else {
emit("Fetching QRZ.com logbook (full — no since date)…")
}
fr, err := extsvc.FetchQRZ(ctx, nil, cfg.QRZ.APIKey, "ALL")
// Ask QRZ for the WINDOW rather than the whole logbook. "ALL" made the
// server serialise every QSO — 56 MB for a 117k-record log — which it
// truncates mid-record, so the parse died two thirds of the way through
// and everything past that point was never seen. AFTER: narrows it at the
// source; the client-side date filter below stays as the backstop.
fetchOpt := "ALL"
if sinceDate != "" {
fetchOpt = "AFTER:" + sinceDate
}
fr, err := extsvc.FetchQRZ(ctx, nil, cfg.QRZ.APIKey, fetchOpt)
if err != nil {
emit("Fetch failed: " + err.Error())
done(matched, total)
@@ -9570,14 +9579,11 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
}
adifText := fr.ADIF
emit(fmt.Sprintf("QRZ RESULT=%s COUNT=%s, ADIF %d bytes", fr.Result, fr.Count, len(adifText)))
// Persist the last-download date NOW (right after a successful fetch),
// not at the end: the QRZ logbook can be huge (tens of thousands of
// records) and the user may close the panel mid-processing — storing it
// late meant the date was never saved, so "since last download" kept
// resolving to empty and re-pulled everything.
if a.settings != nil {
a.setSetting(a.profileScope()+keyExtQRZLastDownload, time.Now().UTC().Format("2006-01-02"))
}
// The last-download date is stored at the END, and ONLY if the whole ADIF
// parsed. It used to be written here, right after the fetch: when the
// response was truncated (see above) the window advanced over records that
// were never processed, and every later run skipped them for good. A
// window that moves past unseen data is worse than one that re-reads.
if snip := strings.TrimSpace(adifText); snip != "" {
if len(snip) > 300 {
snip = snip[:300]
@@ -9709,10 +9715,15 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
sort.Strings(keys)
emit(fmt.Sprintf("Parsed %d record(s). Fields seen: %s", parsed, strings.Join(keys, ", ")))
emit(fmt.Sprintf("Confirmed %d, added %d (of %d returned)", matched, added, total))
if perr != nil {
emit("The download was INCOMPLETE — QRZ.com cut the ADIF short, so the records after that point were not read.")
emit("The last-download date has NOT been moved, so nothing is lost: run it again (a narrower window returns less data).")
} else if a.settings != nil {
a.setSetting(a.profileScope()+keyExtQRZLastDownload, time.Now().UTC().Format("2006-01-02"))
}
if qrzSkippedOtherCall > 0 {
emit(fmt.Sprintf("Skipped %d QSO(s) logged under another of your callsigns (scoped to %s).", qrzSkippedOtherCall, qrzOwner))
}
// (last-download date already stored right after the fetch above)
case extsvc.ServiceEQSL:
sinceDate := resolveSince(keyExtEQSLLastDownload)