rouggy/OpsLog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

fix: Upload to HRDLog

Browse Source
This commit is contained in:
Gregory Salaun
2026-06-18 22:58:00 +02:00
parent 4d074de27e
commit 183db7ac2b
4 changed files with 161 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.go
+77 -43
View File
@@ -5224,65 +5224,99 @@ func (a *App) runManualUpload(svc extsvc.Service, ids []int64, cfg extsvc.Extern
emit(fmt.Sprintf("LoTW: %d QSO(s) uploaded", uploaded))
}
} else if svc == extsvc.ServiceClublog || svc == extsvc.ServiceHRDLog {
// Club Log and HRDLog both accept a whole ADIF document in one request
// and dedupe server-side, so upload in chunks instead of one request per
// QSO. Chunked so a single failure doesn't lose the whole run and the
// user sees progress.
name, chunk := "Club Log", 100
statusCol, dateCol := "clublog_qso_upload_status", "clublog_qso_upload_date"
if svc == extsvc.ServiceHRDLog {
name, chunk = "HRDLog", 50
statusCol, dateCol = "hrdlog_qso_upload_status", "hrdlog_qso_upload_date"
}
type item struct {
id int64
rec string
call string
}
// Fetch the selected QSOs in BULK (chunked IN queries) instead of one
// GetByID per QSO — on a remote MySQL, 25k individual SELECTs is what made
// this crawl.
var items []item
for _, id := range ids {
q, gerr := a.qso.GetByID(ctx, id)
call := ""
if gerr == nil {
call = q.Callsign
const fetchChunk = 1000
for s := 0; s < len(ids); s += fetchChunk {
e := s + fetchChunk
if e > len(ids) {
e = len(ids)
}
rec, ok := a.buildUploadADIF(id, "")
if !ok {
emit(call + " — skipped (no record)")
continue
}
items = append(items, item{id: id, rec: rec, call: call})
_ = a.qso.IterateByIDs(ctx, ids[s:e], func(q qso.QSO) error {
items = append(items, item{id: q.ID, rec: adif.SingleRecordADIF(q), call: q.Callsign})
return nil
})
}
emit(fmt.Sprintf("%s: uploading %d QSO(s) in batches of %d…", name, len(items), chunk))
for start := 0; start < len(items); start += chunk {
end := start + chunk
if end > len(items) {
end = len(items)
date := time.Now().UTC().Format("20060102")
if svc == extsvc.ServiceClublog {
// Club Log accepts a whole ADIF file (putlogs.php) and dedupes
// server-side → upload in chunks, one HTTP request per 100 QSOs.
const chunk = 100
emit(fmt.Sprintf("Club Log: uploading %d QSO(s) in batches of %d…", len(items), chunk))
for start := 0; start < len(items); start += chunk {
end := start + chunk
if end > len(items) {
end = len(items)
}
batch := items[start:end]
recs := make([]string, len(batch))
batchIDs := make([]int64, len(batch))
for i, it := range batch {
recs[i] = it.rec
batchIDs[i] = it.id
}
res, err := extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, adif.BatchRecordsADIF(recs))
if err == nil && res.OK {
if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, batchIDs); merr != nil {
applog.Printf("extsvc: Club Log batch mark: %v", merr)
}
uploaded += len(batch)
emit(fmt.Sprintf("Club Log: %d/%d uploaded", end, len(items)))
} else {
msg := res.Message
if err != nil {
msg = err.Error()
}
emit(fmt.Sprintf("Club Log: batch of %d FAILED: %s", len(batch), msg))
}
}
batch := items[start:end]
recs := make([]string, len(batch))
for i, it := range batch {
recs[i] = it.rec
} else {
// HRDLog's NewEntry.aspx inserts only the FIRST record of a multi-
// record ADIF, so upload ONE record per request. The DB stays cheap:
// bulk fetch above + the marks flushed in batches (not one per QSO).
emit(fmt.Sprintf("HRDLog: uploading %d QSO(s) (one request each)…", len(items)))
var doneIDs []int64
flush := func() {
if len(doneIDs) == 0 {
return
}
if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, doneIDs); merr != nil {
applog.Printf("extsvc: HRDLog batch mark: %v", merr)
}
doneIDs = doneIDs[:0]
}
doc := adif.BatchRecordsADIF(recs)
var res extsvc.UploadResult
var err error
if svc == extsvc.ServiceHRDLog {
res, err = extsvc.UploadHRDLogADIF(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, doc)
} else {
res, err = extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, doc)
}
if err == nil && res.OK {
for _, it := range batch {
a.markExtUploaded(svc, it.id, "")
for i, it := range items {
res, err := extsvc.UploadHRDLog(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, it.rec)
if err == nil && res.OK {
doneIDs = append(doneIDs, it.id)
uploaded++
} else {
msg := res.Message
if err != nil {
msg = err.Error()
}
emit(it.call + " — FAILED: " + msg)
}
emit(fmt.Sprintf("%s: %d/%d uploaded", name, end, len(items)))
} else {
msg := res.Message
if err != nil {
msg = err.Error()
if len(doneIDs) >= 200 {
flush()
}
if (i+1)%50 == 0 || i+1 == len(items) {
emit(fmt.Sprintf("HRDLog: %d/%d uploaded", uploaded, len(items)))
}
emit(fmt.Sprintf("%s: batch of %d FAILED: %s", name, len(batch), msg))
}
flush()
}
} else {
// QRZ.com: one record per request (its logbook API has no batch upload).
frontend/src/components/QSLManagerModal.tsx
+58 -45
View File
@@ -1,5 +1,7 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { UploadCloud, DownloadCloud, Search, Loader2, ScrollText, ListChecks, Trees, ExternalLink } from 'lucide-react';
import { AllCommunityModule, ModuleRegistry, themeQuartz, type ColDef } from 'ag-grid-community';
import { AgGridReact } from 'ag-grid-react';
import { Button } from '@/components/ui/button';
import { Checkbox } from '@/components/ui/checkbox';
import {
@@ -10,11 +12,31 @@ import { FindQSOsForUpload, UploadQSOsManual, DownloadConfirmations, SyncPOTAHun
import { Input } from '@/components/ui/input';
import { EventsOn } from '../../wailsjs/runtime/runtime';
ModuleRegistry.registerModules([AllCommunityModule]);
// Warm theme matching the other grids (Recent QSOs / Cluster).
const qslTheme = themeQuartz.withParams({
fontFamily: 'inherit', fontSize: 12.5, backgroundColor: '#faf6ea', foregroundColor: '#2a2419',
headerBackgroundColor: '#e8dfc9', headerTextColor: '#5a4f3a', headerFontWeight: 600,
oddRowBackgroundColor: '#f5efe0', rowHoverColor: '#ecdcb4', selectedRowBackgroundColor: '#f0d9a8',
borderColor: '#c8b994', rowBorder: { color: '#d8c9a8', width: 1 }, columnBorder: { color: '#d8c9a8', width: 1 },
cellHorizontalPadding: 10, rowHeight: 30, headerHeight: 32, spacing: 4, accentColor: '#b8410c', iconSize: 12,
});
type UploadRow = {
id: number; qso_date: string; callsign: string;
band: string; mode: string; country: string; status: string;
};
const UPLOAD_COLS: ColDef<UploadRow>[] = [
{ field: 'qso_date', headerName: 'Date UTC', valueFormatter: (p) => fmtDate(p.value), minWidth: 150 },
{ field: 'callsign', headerName: 'Callsign', cellClass: 'font-mono font-bold', width: 130 },
{ field: 'band', headerName: 'Band', width: 90 },
{ field: 'mode', headerName: 'Mode', width: 100 },
{ field: 'country', headerName: 'Country', flex: 1, minWidth: 140 },
{ field: 'status', headerName: 'Sent', width: 90, valueFormatter: (p) => p.value || '—' },
];
type Confirmation = {
callsign: string; qso_date: string; band: string; mode: string; country: string;
new_dxcc: boolean; new_band: boolean; new_slot: boolean;
@@ -151,7 +173,10 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi
}
const [sent, setSent] = useState('R');
const [rows, setRows] = useState<UploadRow[]>([]);
const [selected, setSelected] = useState<Set<number>>(new Set());
// Selection lives in the (virtualized) ag-grid — it handles 25k rows smoothly.
const gridRef = useRef<AgGridReact<UploadRow>>(null);
const [selectedCount, setSelectedCount] = useState(0);
const selectAllNext = useRef(false); // selectAll once after the next data load
const [searching, setSearching] = useState(false);
const [error, setError] = useState('');
const [addNotFound, setAddNotFound] = useState(false);
@@ -182,10 +207,20 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi
return () => { offLog(); offDone(); offConf(); };
}, []);
const selectedCount = selected.size;
const allSelected = rows.length > 0 && selected.size === rows.length;
const serviceLabel = useMemo(() => SERVICES.find((s) => s.v === service)?.label ?? service, [service]);
// Grid selection → just track the count; ids are read from the grid at upload.
function onUploadSelChanged() {
setSelectedCount(gridRef.current?.api?.getSelectedNodes()?.length ?? 0);
}
// After "Select required" loads new rows, select them all (the old default).
function onUploadRowsLoaded() {
if (selectAllNext.current) {
selectAllNext.current = false;
gridRef.current?.api?.selectAll();
}
}
const shownConfs = useMemo(() => confirmations.filter((c) => {
switch (confFilter) {
case 'new': return c.new_dxcc || c.new_band || c.new_slot;
@@ -202,32 +237,22 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi
try {
const r: any = await FindQSOsForUpload(service, sent === '_' ? '' : sent);
const list = (r ?? []) as UploadRow[];
selectAllNext.current = true; // pre-select everything once the grid renders
setRows(list);
setSelected(new Set(list.map((x) => x.id)));
setSelectedCount(list.length);
setViewMode('upload');
setShowLog(false);
} catch (e: any) {
setError(String(e?.message ?? e));
setRows([]);
setSelected(new Set());
setSelectedCount(0);
} finally {
setSearching(false);
}
}, [service, sent]);
function toggle(id: number) {
setSelected((s) => {
const n = new Set(s);
if (n.has(id)) n.delete(id); else n.add(id);
return n;
});
}
function toggleAll() {
setSelected(allSelected ? new Set() : new Set(rows.map((r) => r.id)));
}
async function upload() {
const ids = rows.filter((r) => selected.has(r.id)).map((r) => r.id);
const ids = ((gridRef.current?.api?.getSelectedRows() as UploadRow[] | undefined) ?? []).map((r) => r.id);
if (ids.length === 0) return;
setLogLines([]); setBusy(true); setLogAction('upload'); setShowLog(true);
try { await UploadQSOsManual(service, ids); }
@@ -484,33 +509,21 @@ export function QSLManagerPanel({ onEditQSO }: { onEditQSO?: (id: number) => voi
) : rows.length === 0 ? (
<div className="text-sm text-muted-foreground py-10 text-center">Pick a service + sent status, then “Select required”.</div>
) : (
<table className="w-full text-xs border-collapse">
<thead className="sticky top-0 bg-card">
<tr className="text-left text-muted-foreground border-b border-border">
<th className="py-1.5 px-2 w-8"><Checkbox checked={allSelected} onCheckedChange={toggleAll} /></th>
<th className="py-1.5 px-2">Date UTC</th><th className="py-1.5 px-2">Callsign</th>
<th className="py-1.5 px-2">Band</th><th className="py-1.5 px-2">Mode</th>
<th className="py-1.5 px-2">Country</th><th className="py-1.5 px-2">Sent</th>
</tr>
</thead>
<tbody>
{rows.map((r) => (
<tr key={r.id}
className={cn('border-b border-border/40 cursor-pointer hover:bg-accent/30', selected.has(r.id) && 'bg-primary/5')}
onClick={() => toggle(r.id)}>
<td className="py-1 px-2" onClick={(e) => e.stopPropagation()}>
<Checkbox checked={selected.has(r.id)} onCheckedChange={() => toggle(r.id)} />
</td>
<td className="py-1 px-2 font-mono">{fmtDate(r.qso_date)}</td>
<td className="py-1 px-2 font-mono font-bold">{r.callsign}</td>
<td className="py-1 px-2">{r.band}</td>
<td className="py-1 px-2">{r.mode}</td>
<td className="py-1 px-2 text-muted-foreground">{r.country}</td>
<td className="py-1 px-2 font-mono">{r.status || '—'}</td>
</tr>
))}
</tbody>
</table>
<div className="h-full w-full">
<AgGridReact<UploadRow>
ref={gridRef}
theme={qslTheme}
rowData={rows}
columnDefs={UPLOAD_COLS}
defaultColDef={{ sortable: true, resizable: true, filter: true }}
rowSelection={{ mode: 'multiRow', checkboxes: true, headerCheckbox: true }}
onSelectionChanged={onUploadSelChanged}
onRowDataUpdated={onUploadRowsLoaded}
animateRows={false}
suppressCellFocus
getRowId={(p) => String((p.data as any).id)}
/>
</div>
)}
</div>
internal/extsvc/hrdlog.go
+3 -56
View File
@@ -6,7 +6,6 @@ import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
@@ -117,61 +116,9 @@ func UploadHRDLog(ctx context.Context, client *http.Client, callsign, code, adif
return UploadResult{OK: false, Message: reason}, fmt.Errorf("hrdlog: upload failed: %s", reason)
}
// UploadHRDLogADIF pushes a WHOLE ADIF document (header + many records) to
// HRDLog.net in one NewEntry request — HRDLog parses every <eor> record and
// replies "<insert>N" with the number actually inserted (duplicates aren't
// counted but aren't errors). Use this for bulk uploads instead of calling
// UploadHRDLog once per QSO.
func UploadHRDLogADIF(ctx context.Context, client *http.Client, callsign, code, adifDoc string) (UploadResult, error) {
callsign = strings.ToUpper(strings.TrimSpace(callsign))
code = strings.TrimSpace(code)
if callsign == "" {
return UploadResult{}, fmt.Errorf("hrdlog: station callsign not set")
}
if code == "" {
return UploadResult{}, fmt.Errorf("hrdlog: upload code not set")
}
if strings.TrimSpace(adifDoc) == "" {
return UploadResult{}, fmt.Errorf("hrdlog: empty adif")
}
body, err := hrdlogPost(ctx, client, callsign, code, adifDoc)
if err != nil {
return UploadResult{OK: false, Message: body}, err
}
if reason := authErrHRDLog(body); reason != "" {
return UploadResult{OK: false, Message: reason}, fmt.Errorf("hrdlog: %s", reason)
}
if n, ok := parseHRDLogInsert(body); ok {
return UploadResult{OK: true, Message: fmt.Sprintf("%d added", n)}, nil
}
if strings.Contains(strings.ToLower(body), "<error>") {
return UploadResult{OK: false, Message: body}, fmt.Errorf("hrdlog: %s", body)
}
return UploadResult{OK: true, Message: "uploaded"}, nil
}
// parseHRDLogInsert reads N from "<insert>N" (or "<insert>N</insert>").
func parseHRDLogInsert(body string) (int, bool) {
b := strings.ToLower(body)
i := strings.Index(b, "<insert>")
if i < 0 {
return 0, false
}
rest := b[i+len("<insert>"):]
j := 0
for j < len(rest) && rest[j] >= '0' && rest[j] <= '9' {
j++
}
if j == 0 {
return 0, false
}
n, err := strconv.Atoi(rest[:j])
if err != nil {
return 0, false
}
return n, true
}
// NOTE: HRDLog's NewEntry.aspx inserts ONLY the first record of a multi-record
// ADIFData, so there is no batch upload — callers must POST one record per
// request (see UploadHRDLog). The bulk uploader in app.go does exactly that.
// TestHRDLog validates the configured HRDLog credentials with a REAL request:
// it posts an empty ADIF so nothing is inserted, then checks for HRDLog's auth
internal/qso/qso.go
+23
View File
@@ -625,6 +625,29 @@ func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) erro
return nil
}
// MarkUploadedBatch sets <statusCol>='Y' and <dateCol>=date on EVERY id in one
// UPDATE — used by bulk upload (Club Log / HRDLog) so a 25k-QSO run isn't one
// round-trip per QSO on a remote MySQL. statusCol/dateCol come from a fixed
// whitelist (not user input), so the column interpolation is safe.
func (r *Repo) MarkUploadedBatch(ctx context.Context, statusCol, dateCol, date string, ids []int64) error {
if len(ids) == 0 {
return nil
}
ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
args := make([]any, 0, len(ids)+2)
args = append(args, date, db.NowISO())
for _, id := range ids {
args = append(args, id)
}
_, err := r.db.ExecContext(ctx,
`UPDATE qso SET `+statusCol+` = 'Y', `+dateCol+` = ?, updated_at = ? WHERE id IN (`+ph+`)`,
args...)
if err != nil {
return fmt.Errorf("mark uploaded batch (%d): %w", len(ids), err)
}
return nil
}
// MarkEQSLSent stamps EQSL_QSL_SENT=Y and the sent date after a successful
// eQSL e-mail. date is an ADIF YYYYMMDD string.
func (r *Repo) MarkEQSLSent(ctx context.Context, id int64, date string) error {