From 183db7ac2b6e4ff8ee3a250b0624ca3f2a117cb3 Mon Sep 17 00:00:00 2001 From: Gregory Salaun Date: Thu, 18 Jun 2026 22:58:00 +0200 Subject: [PATCH] fix: Upload to HRDLog --- app.go | 120 +++++++++++++------- frontend/src/components/QSLManagerModal.tsx | 103 +++++++++-------- internal/extsvc/hrdlog.go | 59 +--------- internal/qso/qso.go | 23 ++++ 4 files changed, 161 insertions(+), 144 deletions(-) diff --git a/app.go b/app.go index e04b0d3..c7ca3b8 100644 --- a/app.go +++ b/app.go @@ -5224,65 +5224,99 @@ func (a *App) runManualUpload(svc extsvc.Service, ids []int64, cfg extsvc.Extern emit(fmt.Sprintf("LoTW: %d QSO(s) uploaded", uploaded)) } } else if svc == extsvc.ServiceClublog || svc == extsvc.ServiceHRDLog { - // Club Log and HRDLog both accept a whole ADIF document in one request - // and dedupe server-side, so upload in chunks instead of one request per - // QSO. Chunked so a single failure doesn't lose the whole run and the - // user sees progress. - name, chunk := "Club Log", 100 + statusCol, dateCol := "clublog_qso_upload_status", "clublog_qso_upload_date" if svc == extsvc.ServiceHRDLog { - name, chunk = "HRDLog", 50 + statusCol, dateCol = "hrdlog_qso_upload_status", "hrdlog_qso_upload_date" } type item struct { id int64 rec string call string } + // Fetch the selected QSOs in BULK (chunked IN queries) instead of one + // GetByID per QSO — on a remote MySQL, 25k individual SELECTs is what made + // this crawl. var items []item - for _, id := range ids { - q, gerr := a.qso.GetByID(ctx, id) - call := "" - if gerr == nil { - call = q.Callsign + const fetchChunk = 1000 + for s := 0; s < len(ids); s += fetchChunk { + e := s + fetchChunk + if e > len(ids) { + e = len(ids) } - rec, ok := a.buildUploadADIF(id, "") - if !ok { - emit(call + " — skipped (no record)") - continue - } - items = append(items, item{id: id, rec: rec, call: call}) + _ = a.qso.IterateByIDs(ctx, ids[s:e], func(q qso.QSO) error { + items = append(items, item{id: q.ID, rec: adif.SingleRecordADIF(q), call: q.Callsign}) + return nil + }) } - emit(fmt.Sprintf("%s: uploading %d QSO(s) in batches of %d…", name, len(items), chunk)) - for start := 0; start < len(items); start += chunk { - end := start + chunk - if end > len(items) { - end = len(items) + date := time.Now().UTC().Format("20060102") + + if svc == extsvc.ServiceClublog { + // Club Log accepts a whole ADIF file (putlogs.php) and dedupes + // server-side → upload in chunks, one HTTP request per 100 QSOs. + const chunk = 100 + emit(fmt.Sprintf("Club Log: uploading %d QSO(s) in batches of %d…", len(items), chunk)) + for start := 0; start < len(items); start += chunk { + end := start + chunk + if end > len(items) { + end = len(items) + } + batch := items[start:end] + recs := make([]string, len(batch)) + batchIDs := make([]int64, len(batch)) + for i, it := range batch { + recs[i] = it.rec + batchIDs[i] = it.id + } + res, err := extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, adif.BatchRecordsADIF(recs)) + if err == nil && res.OK { + if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, batchIDs); merr != nil { + applog.Printf("extsvc: Club Log batch mark: %v", merr) + } + uploaded += len(batch) + emit(fmt.Sprintf("Club Log: %d/%d uploaded", end, len(items))) + } else { + msg := res.Message + if err != nil { + msg = err.Error() + } + emit(fmt.Sprintf("Club Log: batch of %d FAILED: %s", len(batch), msg)) + } } - batch := items[start:end] - recs := make([]string, len(batch)) - for i, it := range batch { - recs[i] = it.rec + } else { + // HRDLog's NewEntry.aspx inserts only the FIRST record of a multi- + // record ADIF, so upload ONE record per request. The DB stays cheap: + // bulk fetch above + the marks flushed in batches (not one per QSO). + emit(fmt.Sprintf("HRDLog: uploading %d QSO(s) (one request each)…", len(items))) + var doneIDs []int64 + flush := func() { + if len(doneIDs) == 0 { + return + } + if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, doneIDs); merr != nil { + applog.Printf("extsvc: HRDLog batch mark: %v", merr) + } + doneIDs = doneIDs[:0] } - doc := adif.BatchRecordsADIF(recs) - var res extsvc.UploadResult - var err error - if svc == extsvc.ServiceHRDLog { - res, err = extsvc.UploadHRDLogADIF(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, doc) - } else { - res, err = extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, doc) - } - if err == nil && res.OK { - for _, it := range batch { - a.markExtUploaded(svc, it.id, "") + for i, it := range items { + res, err := extsvc.UploadHRDLog(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, it.rec) + if err == nil && res.OK { + doneIDs = append(doneIDs, it.id) uploaded++ + } else { + msg := res.Message + if err != nil { + msg = err.Error() + } + emit(it.call + " — FAILED: " + msg) } - emit(fmt.Sprintf("%s: %d/%d uploaded", name, end, len(items))) - } else { - msg := res.Message - if err != nil { - msg = err.Error() + if len(doneIDs) >= 200 { + flush() + } + if (i+1)%50 == 0 || i+1 == len(items) { + emit(fmt.Sprintf("HRDLog: %d/%d uploaded", uploaded, len(items))) } - emit(fmt.Sprintf("%s: batch of %d FAILED: %s", name, len(batch), msg)) } + flush() } } else { // QRZ.com: one record per request (its logbook API has no batch upload). diff --git a/frontend/src/components/QSLManagerModal.tsx b/frontend/src/components/QSLManagerModal.tsx index 060e5cf..936b6df 100644 --- a/frontend/src/components/QSLManagerModal.tsx +++ b/frontend/src/components/QSLManagerModal.tsx @@ -1,5 +1,7 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { UploadCloud, DownloadCloud, Search, Loader2, ScrollText, ListChecks, Trees, ExternalLink } from 'lucide-react'; +import { AllCommunityModule, ModuleRegistry, themeQuartz, type ColDef } from 'ag-grid-community'; +import { AgGridReact } from 'ag-grid-react'; import { Button } from '@/components/ui/button'; import { Checkbox } from '@/components/ui/checkbox'; import { @@ -10,11 +12,31 @@ import { FindQSOsForUpload, UploadQSOsManual, DownloadConfirmations, SyncPOTAHun import { Input } from '@/components/ui/input'; import { EventsOn } from '../../wailsjs/runtime/runtime'; +ModuleRegistry.registerModules([AllCommunityModule]); + +// Warm theme matching the other grids (Recent QSOs / Cluster). +const qslTheme = themeQuartz.withParams({ + fontFamily: 'inherit', fontSize: 12.5, backgroundColor: '#faf6ea', foregroundColor: '#2a2419', + headerBackgroundColor: '#e8dfc9', headerTextColor: '#5a4f3a', headerFontWeight: 600, + oddRowBackgroundColor: '#f5efe0', rowHoverColor: '#ecdcb4', selectedRowBackgroundColor: '#f0d9a8', + borderColor: '#c8b994', rowBorder: { color: '#d8c9a8', width: 1 }, columnBorder: { color: '#d8c9a8', width: 1 }, + cellHorizontalPadding: 10, rowHeight: 30, headerHeight: 32, spacing: 4, accentColor: '#b8410c', iconSize: 12, +}); + type UploadRow = { id: number; qso_date: string; callsign: string; band: string; mode: string; country: string; status: string; }; +const UPLOAD_COLS: ColDef[] = [ + { field: 'qso_date', headerName: 'Date UTC', valueFormatter: (p) => fmtDate(p.value), minWidth: 150 }, + { field: 'callsign', headerName: 'Callsign', cellClass: 'font-mono font-bold', width: 130 }, + { field: 'band', headerName: 'Band', width: 90 }, + { field: 'mode', headerName: 'Mode', width: 100 }, + { field: 'country', headerName: 'Country', flex: 1, minWidth: 140 }, + { field: 'status', headerName: 'Sent', width: 90, valueFormatter: (p) => p.value || '—' }, +]; + type Confirmation = { callsign: string; qso_date: string; band: string; mode: string; country: string; new_dxcc: boolean; new_band: boolean; new_slot: boolean; @@ -151,7 +173,10 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi } const [sent, setSent] = useState('R'); const [rows, setRows] = useState([]); - const [selected, setSelected] = useState>(new Set()); + // Selection lives in the (virtualized) ag-grid — it handles 25k rows smoothly. + const gridRef = useRef>(null); + const [selectedCount, setSelectedCount] = useState(0); + const selectAllNext = useRef(false); // selectAll once after the next data load const [searching, setSearching] = useState(false); const [error, setError] = useState(''); const [addNotFound, setAddNotFound] = useState(false); @@ -182,10 +207,20 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi return () => { offLog(); offDone(); offConf(); }; }, []); - const selectedCount = selected.size; - const allSelected = rows.length > 0 && selected.size === rows.length; const serviceLabel = useMemo(() => SERVICES.find((s) => s.v === service)?.label ?? service, [service]); + // Grid selection → just track the count; ids are read from the grid at upload. + function onUploadSelChanged() { + setSelectedCount(gridRef.current?.api?.getSelectedNodes()?.length ?? 0); + } + // After "Select required" loads new rows, select them all (the old default). + function onUploadRowsLoaded() { + if (selectAllNext.current) { + selectAllNext.current = false; + gridRef.current?.api?.selectAll(); + } + } + const shownConfs = useMemo(() => confirmations.filter((c) => { switch (confFilter) { case 'new': return c.new_dxcc || c.new_band || c.new_slot; @@ -202,32 +237,22 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi try { const r: any = await FindQSOsForUpload(service, sent === '_' ? '' : sent); const list = (r ?? []) as UploadRow[]; + selectAllNext.current = true; // pre-select everything once the grid renders setRows(list); - setSelected(new Set(list.map((x) => x.id))); + setSelectedCount(list.length); setViewMode('upload'); setShowLog(false); } catch (e: any) { setError(String(e?.message ?? e)); setRows([]); - setSelected(new Set()); + setSelectedCount(0); } finally { setSearching(false); } }, [service, sent]); - function toggle(id: number) { - setSelected((s) => { - const n = new Set(s); - if (n.has(id)) n.delete(id); else n.add(id); - return n; - }); - } - function toggleAll() { - setSelected(allSelected ? new Set() : new Set(rows.map((r) => r.id))); - } - async function upload() { - const ids = rows.filter((r) => selected.has(r.id)).map((r) => r.id); + const ids = ((gridRef.current?.api?.getSelectedRows() as UploadRow[] | undefined) ?? []).map((r) => r.id); if (ids.length === 0) return; setLogLines([]); setBusy(true); setLogAction('upload'); setShowLog(true); try { await UploadQSOsManual(service, ids); } @@ -484,33 +509,21 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi ) : rows.length === 0 ? (
Pick a service + sent status, then “Select required”.
) : ( - - - - - - - - - - - {rows.map((r) => ( - toggle(r.id)}> - - - - - - - - - ))} - -
Date UTCCallsignBandModeCountrySent
e.stopPropagation()}> - toggle(r.id)} /> - {fmtDate(r.qso_date)}{r.callsign}{r.band}{r.mode}{r.country}{r.status || '—'}
+
+ + ref={gridRef} + theme={qslTheme} + rowData={rows} + columnDefs={UPLOAD_COLS} + defaultColDef={{ sortable: true, resizable: true, filter: true }} + rowSelection={{ mode: 'multiRow', checkboxes: true, headerCheckbox: true }} + onSelectionChanged={onUploadSelChanged} + onRowDataUpdated={onUploadRowsLoaded} + animateRows={false} + suppressCellFocus + getRowId={(p) => String((p.data as any).id)} + /> +
)} diff --git a/internal/extsvc/hrdlog.go b/internal/extsvc/hrdlog.go index e485a1d..a5d0ad9 100644 --- a/internal/extsvc/hrdlog.go +++ b/internal/extsvc/hrdlog.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "time" ) @@ -117,61 +116,9 @@ func UploadHRDLog(ctx context.Context, client *http.Client, callsign, code, adif return UploadResult{OK: false, Message: reason}, fmt.Errorf("hrdlog: upload failed: %s", reason) } -// UploadHRDLogADIF pushes a WHOLE ADIF document (header + many records) to -// HRDLog.net in one NewEntry request — HRDLog parses every record and -// replies "N" with the number actually inserted (duplicates aren't -// counted but aren't errors). Use this for bulk uploads instead of calling -// UploadHRDLog once per QSO. -func UploadHRDLogADIF(ctx context.Context, client *http.Client, callsign, code, adifDoc string) (UploadResult, error) { - callsign = strings.ToUpper(strings.TrimSpace(callsign)) - code = strings.TrimSpace(code) - if callsign == "" { - return UploadResult{}, fmt.Errorf("hrdlog: station callsign not set") - } - if code == "" { - return UploadResult{}, fmt.Errorf("hrdlog: upload code not set") - } - if strings.TrimSpace(adifDoc) == "" { - return UploadResult{}, fmt.Errorf("hrdlog: empty adif") - } - - body, err := hrdlogPost(ctx, client, callsign, code, adifDoc) - if err != nil { - return UploadResult{OK: false, Message: body}, err - } - if reason := authErrHRDLog(body); reason != "" { - return UploadResult{OK: false, Message: reason}, fmt.Errorf("hrdlog: %s", reason) - } - if n, ok := parseHRDLogInsert(body); ok { - return UploadResult{OK: true, Message: fmt.Sprintf("%d added", n)}, nil - } - if strings.Contains(strings.ToLower(body), "") { - return UploadResult{OK: false, Message: body}, fmt.Errorf("hrdlog: %s", body) - } - return UploadResult{OK: true, Message: "uploaded"}, nil -} - -// parseHRDLogInsert reads N from "N" (or "N"). -func parseHRDLogInsert(body string) (int, bool) { - b := strings.ToLower(body) - i := strings.Index(b, "") - if i < 0 { - return 0, false - } - rest := b[i+len(""):] - j := 0 - for j < len(rest) && rest[j] >= '0' && rest[j] <= '9' { - j++ - } - if j == 0 { - return 0, false - } - n, err := strconv.Atoi(rest[:j]) - if err != nil { - return 0, false - } - return n, true -} +// NOTE: HRDLog's NewEntry.aspx inserts ONLY the first record of a multi-record +// ADIFData, so there is no batch upload — callers must POST one record per +// request (see UploadHRDLog). The bulk uploader in app.go does exactly that. // TestHRDLog validates the configured HRDLog credentials with a REAL request: // it posts an empty ADIF so nothing is inserted, then checks for HRDLog's auth diff --git a/internal/qso/qso.go b/internal/qso/qso.go index 982b1a9..b9922ee 100644 --- a/internal/qso/qso.go +++ b/internal/qso/qso.go @@ -625,6 +625,29 @@ func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) erro return nil } +// MarkUploadedBatch sets ='Y' and =date on EVERY id in one +// UPDATE — used by bulk upload (Club Log / HRDLog) so a 25k-QSO run isn't one +// round-trip per QSO on a remote MySQL. statusCol/dateCol come from a fixed +// whitelist (not user input), so the column interpolation is safe. +func (r *Repo) MarkUploadedBatch(ctx context.Context, statusCol, dateCol, date string, ids []int64) error { + if len(ids) == 0 { + return nil + } + ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + args := make([]any, 0, len(ids)+2) + args = append(args, date, db.NowISO()) + for _, id := range ids { + args = append(args, id) + } + _, err := r.db.ExecContext(ctx, + `UPDATE qso SET `+statusCol+` = 'Y', `+dateCol+` = ?, updated_at = ? WHERE id IN (`+ph+`)`, + args...) + if err != nil { + return fmt.Errorf("mark uploaded batch (%d): %w", len(ids), err) + } + return nil +} + // MarkEQSLSent stamps EQSL_QSL_SENT=Y and the sent date after a successful // eQSL e-mail. date is an ADIF YYYYMMDD string. func (r *Repo) MarkEQSLSent(ctx context.Context, id int64, date string) error {