fix: Upload to HRDLog
This commit is contained in:
@@ -5224,65 +5224,99 @@ func (a *App) runManualUpload(svc extsvc.Service, ids []int64, cfg extsvc.Extern
|
||||
emit(fmt.Sprintf("LoTW: %d QSO(s) uploaded", uploaded))
|
||||
}
|
||||
} else if svc == extsvc.ServiceClublog || svc == extsvc.ServiceHRDLog {
|
||||
// Club Log and HRDLog both accept a whole ADIF document in one request
|
||||
// and dedupe server-side, so upload in chunks instead of one request per
|
||||
// QSO. Chunked so a single failure doesn't lose the whole run and the
|
||||
// user sees progress.
|
||||
name, chunk := "Club Log", 100
|
||||
statusCol, dateCol := "clublog_qso_upload_status", "clublog_qso_upload_date"
|
||||
if svc == extsvc.ServiceHRDLog {
|
||||
name, chunk = "HRDLog", 50
|
||||
statusCol, dateCol = "hrdlog_qso_upload_status", "hrdlog_qso_upload_date"
|
||||
}
|
||||
type item struct {
|
||||
id int64
|
||||
rec string
|
||||
call string
|
||||
}
|
||||
// Fetch the selected QSOs in BULK (chunked IN queries) instead of one
|
||||
// GetByID per QSO — on a remote MySQL, 25k individual SELECTs is what made
|
||||
// this crawl.
|
||||
var items []item
|
||||
for _, id := range ids {
|
||||
q, gerr := a.qso.GetByID(ctx, id)
|
||||
call := ""
|
||||
if gerr == nil {
|
||||
call = q.Callsign
|
||||
const fetchChunk = 1000
|
||||
for s := 0; s < len(ids); s += fetchChunk {
|
||||
e := s + fetchChunk
|
||||
if e > len(ids) {
|
||||
e = len(ids)
|
||||
}
|
||||
rec, ok := a.buildUploadADIF(id, "")
|
||||
if !ok {
|
||||
emit(call + " — skipped (no record)")
|
||||
continue
|
||||
}
|
||||
items = append(items, item{id: id, rec: rec, call: call})
|
||||
_ = a.qso.IterateByIDs(ctx, ids[s:e], func(q qso.QSO) error {
|
||||
items = append(items, item{id: q.ID, rec: adif.SingleRecordADIF(q), call: q.Callsign})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
emit(fmt.Sprintf("%s: uploading %d QSO(s) in batches of %d…", name, len(items), chunk))
|
||||
for start := 0; start < len(items); start += chunk {
|
||||
end := start + chunk
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
date := time.Now().UTC().Format("20060102")
|
||||
|
||||
if svc == extsvc.ServiceClublog {
|
||||
// Club Log accepts a whole ADIF file (putlogs.php) and dedupes
|
||||
// server-side → upload in chunks, one HTTP request per 100 QSOs.
|
||||
const chunk = 100
|
||||
emit(fmt.Sprintf("Club Log: uploading %d QSO(s) in batches of %d…", len(items), chunk))
|
||||
for start := 0; start < len(items); start += chunk {
|
||||
end := start + chunk
|
||||
if end > len(items) {
|
||||
end = len(items)
|
||||
}
|
||||
batch := items[start:end]
|
||||
recs := make([]string, len(batch))
|
||||
batchIDs := make([]int64, len(batch))
|
||||
for i, it := range batch {
|
||||
recs[i] = it.rec
|
||||
batchIDs[i] = it.id
|
||||
}
|
||||
res, err := extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, adif.BatchRecordsADIF(recs))
|
||||
if err == nil && res.OK {
|
||||
if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, batchIDs); merr != nil {
|
||||
applog.Printf("extsvc: Club Log batch mark: %v", merr)
|
||||
}
|
||||
uploaded += len(batch)
|
||||
emit(fmt.Sprintf("Club Log: %d/%d uploaded", end, len(items)))
|
||||
} else {
|
||||
msg := res.Message
|
||||
if err != nil {
|
||||
msg = err.Error()
|
||||
}
|
||||
emit(fmt.Sprintf("Club Log: batch of %d FAILED: %s", len(batch), msg))
|
||||
}
|
||||
}
|
||||
batch := items[start:end]
|
||||
recs := make([]string, len(batch))
|
||||
for i, it := range batch {
|
||||
recs[i] = it.rec
|
||||
} else {
|
||||
// HRDLog's NewEntry.aspx inserts only the FIRST record of a multi-
|
||||
// record ADIF, so upload ONE record per request. The DB stays cheap:
|
||||
// bulk fetch above + the marks flushed in batches (not one per QSO).
|
||||
emit(fmt.Sprintf("HRDLog: uploading %d QSO(s) (one request each)…", len(items)))
|
||||
var doneIDs []int64
|
||||
flush := func() {
|
||||
if len(doneIDs) == 0 {
|
||||
return
|
||||
}
|
||||
if merr := a.qso.MarkUploadedBatch(ctx, statusCol, dateCol, date, doneIDs); merr != nil {
|
||||
applog.Printf("extsvc: HRDLog batch mark: %v", merr)
|
||||
}
|
||||
doneIDs = doneIDs[:0]
|
||||
}
|
||||
doc := adif.BatchRecordsADIF(recs)
|
||||
var res extsvc.UploadResult
|
||||
var err error
|
||||
if svc == extsvc.ServiceHRDLog {
|
||||
res, err = extsvc.UploadHRDLogADIF(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, doc)
|
||||
} else {
|
||||
res, err = extsvc.UploadClublogADIF(ctx, nil, cfg.Clublog, doc)
|
||||
}
|
||||
if err == nil && res.OK {
|
||||
for _, it := range batch {
|
||||
a.markExtUploaded(svc, it.id, "")
|
||||
for i, it := range items {
|
||||
res, err := extsvc.UploadHRDLog(ctx, nil, cfg.HRDLog.Callsign, cfg.HRDLog.Code, it.rec)
|
||||
if err == nil && res.OK {
|
||||
doneIDs = append(doneIDs, it.id)
|
||||
uploaded++
|
||||
} else {
|
||||
msg := res.Message
|
||||
if err != nil {
|
||||
msg = err.Error()
|
||||
}
|
||||
emit(it.call + " — FAILED: " + msg)
|
||||
}
|
||||
emit(fmt.Sprintf("%s: %d/%d uploaded", name, end, len(items)))
|
||||
} else {
|
||||
msg := res.Message
|
||||
if err != nil {
|
||||
msg = err.Error()
|
||||
if len(doneIDs) >= 200 {
|
||||
flush()
|
||||
}
|
||||
if (i+1)%50 == 0 || i+1 == len(items) {
|
||||
emit(fmt.Sprintf("HRDLog: %d/%d uploaded", uploaded, len(items)))
|
||||
}
|
||||
emit(fmt.Sprintf("%s: batch of %d FAILED: %s", name, len(batch), msg))
|
||||
}
|
||||
flush()
|
||||
}
|
||||
} else {
|
||||
// QRZ.com: one record per request (its logbook API has no batch upload).
|
||||
|
||||
Reference in New Issue
Block a user