fix: QRZ confirmation download read a third of a large logbook, then skipped the rest for good
Three faults compounding, all visible in one operator's report (117k QSOs): 1. We asked for OPTION=ALL every time, so QRZ serialised the entire logbook — 56 MB — and cut it short. The parser stopped at record 27832 with "unexpected EOF" and everything past that was never read. The window is now sent to the server (OPTION=AFTER:date); the client-side date filter stays as a backstop. 2. The last-download date was stored RIGHT AFTER the fetch, before knowing whether the records had been processed. So the truncated run still advanced the window past data it had never read — and every later run skipped it. It is now written at the end, and only when the ADIF parsed cleanly. A window that moves past unseen data is worse than one that re-reads. 3. Which is why the run in the report ended "Confirmed 0, added 0 (of 0 returned)" despite parsing 27832 records: an earlier truncated run had already pushed the window to yesterday, so every record was filtered out client-side. The operator could not recover without clearing the key by hand. A truncated download now says so in plain words, and says the window was not moved.
This commit is contained in:
@@ -95,11 +95,11 @@ const (
|
||||
keyListsRSTCW = "lists.rst_cw"
|
||||
keyListsRSTDigital = "lists.rst_digital"
|
||||
|
||||
keyCATEnabled = "cat.enabled"
|
||||
keyCATBackend = "cat.backend" // "omnirig" | "flex"
|
||||
keyCATOmniRigNum = "cat.omnirig.rig" // 1 or 2
|
||||
keyCATEnabled = "cat.enabled"
|
||||
keyCATBackend = "cat.backend" // "omnirig" | "flex"
|
||||
keyCATOmniRigNum = "cat.omnirig.rig" // 1 or 2
|
||||
// Which VFO to believe when OmniRig names one. "" = trust the rig file.
|
||||
keyCATOmniRigVFO = "cat.omnirig.vfo" // "" | "A" | "B"
|
||||
keyCATOmniRigVFO = "cat.omnirig.vfo" // "" | "A" | "B"
|
||||
keyCATFlexHost = "cat.flex.host" // FlexRadio IP (native backend)
|
||||
keyCATFlexPort = "cat.flex.port" // FlexRadio TCP port (default 4992)
|
||||
keyCATFlexSpots = "cat.flex.spots" // push cluster spots to the panadapter
|
||||
@@ -329,14 +329,14 @@ type QSLDefaults struct {
|
||||
// CATSettings is the user-tweakable rig-control configuration. Stored as
|
||||
// individual key/value pairs to keep the settings table flat.
|
||||
type CATSettings struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Backend string `json:"backend"` // "omnirig" | "flex" | "icom" | "icom-net" | "tci"
|
||||
OmniRigNum int `json:"omnirig_rig"` // 1 or 2 (OmniRig "Rig1"/"Rig2" slot)
|
||||
Enabled bool `json:"enabled"`
|
||||
Backend string `json:"backend"` // "omnirig" | "flex" | "icom" | "icom-net" | "tci"
|
||||
OmniRigNum int `json:"omnirig_rig"` // 1 or 2 (OmniRig "Rig1"/"Rig2" slot)
|
||||
// OmniRigVFO overrides which VFO OpsLog reads: "" follows what the rig file
|
||||
// reports, "A"/"B" force one. Needed because that report is only as good as
|
||||
// the .ini: an IC-7610 file was seen declaring VFO B permanently while the
|
||||
// operator worked on the main VFO, so OpsLog wrote to A and read B.
|
||||
OmniRigVFO string `json:"omnirig_vfo"` // "" | "A" | "B"
|
||||
OmniRigVFO string `json:"omnirig_vfo"` // "" | "A" | "B"
|
||||
FlexHost string `json:"flex_host"` // FlexRadio IP (native backend)
|
||||
FlexPort int `json:"flex_port"` // FlexRadio TCP port (default 4992)
|
||||
FlexSpots bool `json:"flex_spots"` // push cluster spots to the panadapter
|
||||
@@ -9562,7 +9562,16 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
|
||||
} else {
|
||||
emit("Fetching QRZ.com logbook (full — no since date)…")
|
||||
}
|
||||
fr, err := extsvc.FetchQRZ(ctx, nil, cfg.QRZ.APIKey, "ALL")
|
||||
// Ask QRZ for the WINDOW rather than the whole logbook. "ALL" made the
|
||||
// server serialise every QSO — 56 MB for a 117k-record log — which it
|
||||
// truncates mid-record, so the parse died two thirds of the way through
|
||||
// and everything past that point was never seen. AFTER: narrows it at the
|
||||
// source; the client-side date filter below stays as the backstop.
|
||||
fetchOpt := "ALL"
|
||||
if sinceDate != "" {
|
||||
fetchOpt = "AFTER:" + sinceDate
|
||||
}
|
||||
fr, err := extsvc.FetchQRZ(ctx, nil, cfg.QRZ.APIKey, fetchOpt)
|
||||
if err != nil {
|
||||
emit("Fetch failed: " + err.Error())
|
||||
done(matched, total)
|
||||
@@ -9570,14 +9579,11 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
|
||||
}
|
||||
adifText := fr.ADIF
|
||||
emit(fmt.Sprintf("QRZ RESULT=%s COUNT=%s, ADIF %d bytes", fr.Result, fr.Count, len(adifText)))
|
||||
// Persist the last-download date NOW (right after a successful fetch),
|
||||
// not at the end: the QRZ logbook can be huge (tens of thousands of
|
||||
// records) and the user may close the panel mid-processing — storing it
|
||||
// late meant the date was never saved, so "since last download" kept
|
||||
// resolving to empty and re-pulled everything.
|
||||
if a.settings != nil {
|
||||
a.setSetting(a.profileScope()+keyExtQRZLastDownload, time.Now().UTC().Format("2006-01-02"))
|
||||
}
|
||||
// The last-download date is stored at the END, and ONLY if the whole ADIF
|
||||
// parsed. It used to be written here, right after the fetch: when the
|
||||
// response was truncated (see above) the window advanced over records that
|
||||
// were never processed, and every later run skipped them for good. A
|
||||
// window that moves past unseen data is worse than one that re-reads.
|
||||
if snip := strings.TrimSpace(adifText); snip != "" {
|
||||
if len(snip) > 300 {
|
||||
snip = snip[:300]
|
||||
@@ -9709,10 +9715,15 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe
|
||||
sort.Strings(keys)
|
||||
emit(fmt.Sprintf("Parsed %d record(s). Fields seen: %s", parsed, strings.Join(keys, ", ")))
|
||||
emit(fmt.Sprintf("Confirmed %d, added %d (of %d returned)", matched, added, total))
|
||||
if perr != nil {
|
||||
emit("The download was INCOMPLETE — QRZ.com cut the ADIF short, so the records after that point were not read.")
|
||||
emit("The last-download date has NOT been moved, so nothing is lost: run it again (a narrower window returns less data).")
|
||||
} else if a.settings != nil {
|
||||
a.setSetting(a.profileScope()+keyExtQRZLastDownload, time.Now().UTC().Format("2006-01-02"))
|
||||
}
|
||||
if qrzSkippedOtherCall > 0 {
|
||||
emit(fmt.Sprintf("Skipped %d QSO(s) logged under another of your callsigns (scoped to %s).", qrzSkippedOtherCall, qrzOwner))
|
||||
}
|
||||
// (last-download date already stored right after the fetch above)
|
||||
|
||||
case extsvc.ServiceEQSL:
|
||||
sinceDate := resolveSince(keyExtEQSLLastDownload)
|
||||
|
||||
+4
-2
@@ -19,7 +19,8 @@
|
||||
"Edit QSO: the QRZ/HamQTH fetch button now bypasses the cache, so it really re-reads the callsign — upgrading a QRZ subscription used to change nothing until the cached entry expired a month later. It also no longer blanks an existing value when the lookup comes back with that field empty.",
|
||||
"The diagnostic log no longer carries the FlexRadio meter readings, which drowned everything else.",
|
||||
"OmniRig: a new \"VFO to read\" setting. OmniRig reports the active VFO from the rig file, and some files get it wrong — an IC-7610 file declared VFO B while the operator was on the main VFO, so OpsLog wrote to one VFO and read the other, and the frequency looked frozen. Force VFO A or B when that happens.",
|
||||
"Bulk edit: fixed \"field is not bulk-editable\" on the QRZ.com received status and the confirmation dates added in this version — they were declared in the wrong place and refused at Apply. Bulk-editing State and County, offered since the start, was refused the same way and now works."
|
||||
"Bulk edit: fixed \"field is not bulk-editable\" on the QRZ.com received status and the confirmation dates added in this version — they were declared in the wrong place and refused at Apply. Bulk-editing State and County, offered since the start, was refused the same way and now works.",
|
||||
"QRZ.com confirmation download: the whole logbook was requested every time, which QRZ cuts short on a large log — the read stopped two thirds of the way through and the rest was never seen. Only the window is requested now. And the last-download date is no longer moved when the download was incomplete: it used to advance over records that had never been read, which skipped them for good."
|
||||
],
|
||||
"fr": [
|
||||
"Statistiques : le graphique d'activité suit désormais la période choisie, et Jour / Semaine / Mois / Année en choisissent le pas (les QSO par semaine sur toute la période, par exemple). Il affichait auparavant des fenêtres fixes — les 7 derniers jours, les 30 derniers — quelle que soit la période.",
|
||||
@@ -38,7 +39,8 @@
|
||||
"Édition QSO : le bouton de récupération QRZ/HamQTH contourne désormais le cache et relit donc réellement l'indicatif — passer à un abonnement QRZ payant ne changeait rien tant que l'entrée en cache n'avait pas expiré un mois plus tard. Il n'efface plus non plus une valeur existante lorsque la recherche revient sans ce champ.",
|
||||
"Le journal de diagnostic ne contient plus les mesures des instruments FlexRadio, qui noyaient tout le reste.",
|
||||
"OmniRig : nouveau réglage « VFO à lire ». OmniRig indique le VFO actif d'après le fichier radio, et certains fichiers se trompent — un fichier IC-7610 déclarait le VFO B alors que l'opérateur était sur le VFO principal, si bien qu'OpsLog écrivait dans l'un et lisait l'autre, et la fréquence semblait figée. Forcez le VFO A ou B le cas échéant.",
|
||||
"Édition en lot : correction du refus « field is not bulk-editable » sur le statut de réception QRZ.com et les dates de confirmation ajoutées dans cette version — ils étaient déclarés au mauvais endroit et rejetés au moment d'appliquer. L'édition en lot de State et County, proposée depuis toujours, était refusée de la même façon et fonctionne désormais."
|
||||
"Édition en lot : correction du refus « field is not bulk-editable » sur le statut de réception QRZ.com et les dates de confirmation ajoutées dans cette version — ils étaient déclarés au mauvais endroit et rejetés au moment d'appliquer. L'édition en lot de State et County, proposée depuis toujours, était refusée de la même façon et fonctionne désormais.",
|
||||
"Téléchargement des confirmations QRZ.com : le carnet entier était demandé à chaque fois, ce que QRZ tronque sur un grand log — la lecture s'arrêtait aux deux tiers et le reste n'était jamais vu. Seule la fenêtre est désormais demandée. Et la date du dernier téléchargement n'avance plus lorsque le téléchargement est incomplet : elle passait par-dessus des enregistrements jamais lus, qui étaient alors ignorés définitivement."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user