chore: release v0.25.8

2026-08-17 13:18:32 +02:00
parent 0bab7f05b9
commit 7be6f64596
10 changed files with 661 additions and 83 deletions
@@ -117,6 +117,13 @@ func UploadClublogADIF(ctx context.Context, client *http.Client, cfg ServiceConf
 	if api == "" {
 		api = clublogAppAPIKey
 	}
+	// putlogs.php reads the upload as an ADIF *file*, so it needs a header.
+	// Callers that already build a full document (the QSL Manager) pass one;
+	// callers that only have <EOR>-terminated records (the on-close flush) do
+	// not, and a headerless file is rejected. Same rule as the LoTW writer.
+	if !strings.Contains(strings.ToUpper(adifDoc), "<EOH>") {
+		adifDoc = "OpsLog Club Log upload\n<PROGRAMID:6>OpsLog <EOH>\n" + adifDoc
+	}

 	var buf bytes.Buffer
 	mw := multipart.NewWriter(&buf)
@@ -267,6 +267,78 @@ func UploadEQSL(ctx context.Context, client *http.Client, user, pswd, qthNick, a
 	return UploadResult{OK: false, Message: reason}, fmt.Errorf("eqsl: upload failed: %s", reason)
 }

+// eqslBatchMax is the largest number of records eQSL asks a single upload to
+// carry ("upload only files smaller than about 1000 records at a time", eQSL's
+// own ImportADIF interface notes). Callers chunk to this.
+const eqslBatchMax = 1000
+
+// UploadEQSLBatch pushes MANY ADIF records to eQSL.cc in ONE request.
+//
+// ImportADIF.cfm is a file importer, not a per-QSO endpoint: it takes one *or
+// more* QSOs and answers "Result: X out of Y records added" — the plural in its
+// own reply. So an on-close sweep or a bulk upload is one request, not one per
+// contact. No ADIF header is prepended: the single-record path has always posted
+// bare <EOR> records and eQSL accepts them (per ADIF, a file starting with '<'
+// has no header), and there is no reason to change what is known to work.
+//
+// A PARTIAL result ("97 out of 100") sets Ignored so the caller can say so.
+// eQSL does not identify which records it left out, and in practice they are
+// QSOs it already holds — the same duplicate that UploadEQSL reports as success.
+func UploadEQSLBatch(ctx context.Context, client *http.Client, user, pswd, qthNick string, records []string) (UploadResult, error) {
+	user = strings.ToUpper(strings.TrimSpace(user))
+	if user == "" {
+		return UploadResult{}, fmt.Errorf("eqsl: username (callsign) not set")
+	}
+	if strings.TrimSpace(pswd) == "" {
+		return UploadResult{}, fmt.Errorf("eqsl: password not set")
+	}
+	docs := make([]string, 0, len(records))
+	for _, r := range records {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		docs = append(docs, eqslRecordWithNickname(strings.TrimRight(r, "\r\n"), qthNick))
+	}
+	if len(docs) == 0 {
+		return UploadResult{}, fmt.Errorf("eqsl: empty adif batch")
+	}
+	if len(docs) > eqslBatchMax {
+		return UploadResult{}, fmt.Errorf("eqsl: batch of %d exceeds the %d-record limit", len(docs), eqslBatchMax)
+	}
+
+	body, err := eqslPost(ctx, client, user, pswd, strings.Join(docs, "\n"))
+	if err != nil {
+		return UploadResult{OK: false, Message: body}, err
+	}
+	if reason := authErrEQSL(body); reason != "" {
+		return UploadResult{OK: false, Message: reason}, fmt.Errorf("eqsl: %s", reason)
+	}
+
+	// The counted result is read FIRST here, unlike the single-record path: a
+	// batch reply routinely carries both "Result: 97 out of 100 records added"
+	// and a "Bad record: Duplicate" line for the other three, and matching the
+	// duplicate first would throw away the count that says the rest went in.
+	if m := eqslResultRe.FindStringSubmatch(body); m != nil {
+		added, _ := strconv.Atoi(m[1])
+		total, _ := strconv.Atoi(m[2])
+		if added >= 1 {
+			return UploadResult{OK: true, Message: strings.TrimSpace(m[0]), Ignored: added < total}, nil
+		}
+		// "0 out of N" — nothing added. A re-upload of QSOs eQSL already holds
+		// lands here, and that is not a failure.
+		if strings.Contains(strings.ToLower(body), "duplicate") {
+			return UploadResult{OK: true, Message: "already in logbook"}, nil
+		}
+		reason := eqslReason(body)
+		return UploadResult{OK: false, Message: reason}, fmt.Errorf("eqsl: batch upload failed: %s", reason)
+	}
+	if strings.Contains(strings.ToLower(body), "duplicate") {
+		return UploadResult{OK: true, Message: "already in logbook"}, nil
+	}
+	reason := eqslReason(body)
+	return UploadResult{OK: false, Message: reason}, fmt.Errorf("eqsl: batch upload failed: %s", reason)
+}
+
 // eqslReason trims an eQSL reply to a short human-readable reason: the first
 // "Error:" / "Warning:" / "Bad record:" line if present, else the whole body
 // (capped), else a generic phrase.
@@ -295,9 +295,10 @@ func (m *Manager) CloseUploadCount() int {
 }

 // FlushOnClose uploads every QSO due for an on-close push, scanning the whole
-// logbook (not just this session). Called from the shutdown sequence. QRZ/Club
-// Log go one-by-one (fast HTTP); LoTW is signed and uploaded as a single TQSL
-// batch. Returns the number of QSOs uploaded successfully.
+// logbook (not just this session). Called from the shutdown sequence. QRZ and
+// the rest go one-by-one (fast HTTP, no batch API); LoTW is signed and uploaded
+// as a single TQSL batch, and Club Log goes through its batch endpoint.
+// Returns the number of QSOs uploaded successfully.
 func (m *Manager) FlushOnClose() int {
 	if m.deps.CloseUploadIDs == nil {
 		return 0
@@ -312,41 +313,190 @@ func (m *Manager) FlushOnClose() int {
 		switch svc {
 		case ServiceLoTW:
 			uploaded += m.flushLoTWBatch(ids, cfg.LoTW)
-		case ServiceQRZ:
-			for _, id := range ids {
-				if ok, _ := m.upload(svc, id, cfg.QRZ); ok {
-					uploaded++
-				}
-			}
 		case ServiceClublog:
-			for _, id := range ids {
-				if ok, _ := m.upload(svc, id, cfg.Clublog); ok {
-					uploaded++
-				}
-			}
-		case ServiceHRDLog:
-			for _, id := range ids {
-				if ok, _ := m.upload(svc, id, cfg.HRDLog); ok {
-					uploaded++
-				}
-			}
+			uploaded += m.flushClublogBatch(ids, cfg.Clublog)
 		case ServiceEQSL:
-			for _, id := range ids {
-				if ok, _ := m.upload(svc, id, cfg.EQSL); ok {
-					uploaded++
-				}
-			}
+			uploaded += m.flushEQSLBatch(ids, cfg.EQSL)
+		case ServiceQRZ:
+			uploaded += m.flushOneByOne(svc, ids, cfg.QRZ)
+		case ServiceHRDLog:
+			uploaded += m.flushOneByOne(svc, ids, cfg.HRDLog)
 		case ServiceCloudlog:
-			for _, id := range ids {
-				if ok, _ := m.upload(svc, id, cfg.Cloudlog); ok {
-					uploaded++
-				}
-			}
+			uploaded += m.flushOneByOne(svc, ids, cfg.Cloudlog)
 		}
 	}
 	return uploaded
 }

+// uploadPace is the shortest gap between two consecutive single-QSO uploads in
+// an on-close sweep. QRZ, HRDLog and Cloudlog have no batch endpoint — HRDLog's
+// NewEntry.aspx keeps only the first record of a multi-record ADIF — so a sweep
+// of a freshly imported log is unavoidably one request per contact. It does not
+// have to arrive as fast as the link allows, though: that burst is what a
+// service reads as a robot, and what got an operator's IP threatened at Club Log
+// (see flushClublogBatch). The gap costs nothing in practice, since a round trip
+// to any of these already takes longer than it.
+const uploadPace = 200 * time.Millisecond
+
+// flushOneByOne uploads ids one request at a time, paced. For the services that
+// have no batch API; everything else has its own flush<Service>Batch.
+func (m *Manager) flushOneByOne(svc Service, ids []int64, cfg ServiceConfig) int {
+	uploaded := 0
+	for i, id := range ids {
+		if i > 0 {
+			time.Sleep(uploadPace)
+		}
+		if ok, _ := m.upload(svc, id, cfg); ok {
+			uploaded++
+		}
+	}
+	return uploaded
+}
+
+// eqslBatchChunk is how many QSOs go into one ImportADIF.cfm request. eQSL's own
+// limit is ten times this (eqslBatchMax); the smaller chunk keeps one refused
+// record from taking a thousand others down with it, and keeps the form body
+// small enough to be unremarkable.
+const eqslBatchChunk = 100
+
+// flushEQSLBatch uploads the on-close eQSL QSOs through ImportADIF.cfm in
+// batches instead of one request per contact. Same reasoning as
+// flushClublogBatch — eQSL's import endpoint has always taken a whole file, so
+// the one-at-a-time loop was making hundreds of requests it never needed to.
+func (m *Manager) flushEQSLBatch(ids []int64, cfg ServiceConfig) int {
+	uploaded := 0
+	var records []string
+	var kept []int64
+
+	send := func() {
+		if len(records) == 0 {
+			return
+		}
+		// nil client: UploadEQSLBatch then builds one with a 30 s timeout rather
+		// than reusing the 20 s budget of a single realtime QSO.
+		res, err := UploadEQSLBatch(context.Background(), nil, cfg.Username, cfg.Password, cfg.QTHNickname, records)
+		if err != nil || !res.OK {
+			if err == nil {
+				err = errFromResult(res)
+			}
+			m.logf("extsvc: eqsl batch upload (%d QSOs) failed: %v", len(kept), err)
+			if m.deps.NotifyError != nil {
+				m.deps.NotifyError(ServiceEQSL, 0, err)
+			}
+		} else {
+			// res.Ignored means eQSL took the file but left records out. Say the
+			// count out loud: the whole chunk is still marked sent (eQSL never
+			// says WHICH it dropped, and in practice they are QSOs it already
+			// had), so the log line is the only trace of the shortfall.
+			if res.Ignored {
+				m.logf("extsvc: eqsl batch upload PARTIAL (%d QSOs sent) %s", len(kept), res.Message)
+			} else {
+				m.logf("extsvc: eqsl batch upload OK (%d QSOs) %s", len(kept), res.Message)
+			}
+			if m.deps.MarkUploaded != nil {
+				for _, id := range kept {
+					m.deps.MarkUploaded(ServiceEQSL, id, res.LogID)
+				}
+			}
+			uploaded += len(kept)
+		}
+		records = records[:0]
+		kept = kept[:0]
+	}
+
+	for _, id := range ids {
+		if m.deps.ShouldUpload != nil && !m.deps.ShouldUpload(ServiceEQSL, id) {
+			continue
+		}
+		// eQSL keeps the QSO's own station call; the account is identified by the
+		// credentials and the optional QTH nickname — as in upload().
+		rec, ok := m.deps.BuildADIF(id, "")
+		if !ok {
+			continue
+		}
+		records = append(records, rec)
+		kept = append(kept, id)
+		if len(records) >= eqslBatchChunk {
+			send()
+		}
+	}
+	send()
+	return uploaded
+}
+
+// clublogBatchChunk is how many QSOs go into one putlogs.php request. Club Log
+// dedupes server-side, so chunking is not about correctness — it keeps a single
+// malformed record from failing a whole ten-thousand-QSO document, and matches
+// what the QSL Manager's bulk upload already uses.
+const clublogBatchChunk = 100
+
+// flushClublogBatch uploads the on-close Club Log QSOs through the BATCH
+// endpoint (putlogs.php) rather than one realtime.php call each.
+//
+// It used to walk the ids and call UploadClublog per QSO. On-close upload sweeps
+// the WHOLE logbook, so importing an ADIF — or simply switching Club Log on over
+// an existing log — turned one app close into hundreds of realtime.php posts.
+// That endpoint is reserved for an operator logging contacts as they work them,
+// and Club Log blocks the IP of anything that batches through it: an OpsLog user
+// was flagged by G7VJR for 185 QSOs in four minutes, which is this loop, not a
+// pile-up. Batch upload is the mechanism Club Log provides for exactly this.
+func (m *Manager) flushClublogBatch(ids []int64, cfg ServiceConfig) int {
+	uploaded := 0
+	var records []string
+	var kept []int64
+
+	send := func() {
+		if len(records) == 0 {
+			return
+		}
+		// nil client on purpose: UploadClublogADIF then builds one with a 120 s
+		// timeout. m.deps.Client is the 20 s budget of a single realtime QSO,
+		// which a hundred-QSO document on a slow link would blow through.
+		res, err := UploadClublogADIF(context.Background(), nil, cfg, strings.Join(records, "\n"))
+		if err != nil || !res.OK {
+			if err == nil {
+				err = errFromResult(res)
+			}
+			m.logf("extsvc: clublog batch upload (%d QSOs) failed: %v", len(kept), err)
+			if m.deps.NotifyError != nil {
+				m.deps.NotifyError(ServiceClublog, 0, err)
+			}
+		} else {
+			m.logf("extsvc: clublog batch upload OK (%d QSOs) %s", len(kept), res.Message)
+			if m.deps.MarkUploaded != nil {
+				for _, id := range kept {
+					m.deps.MarkUploaded(ServiceClublog, id, res.LogID)
+				}
+			}
+			uploaded += len(kept)
+		}
+		records = records[:0]
+		kept = kept[:0]
+	}
+
+	for _, id := range ids {
+		// Skip QSOs not eligible (already sent). The wrong-logbook guard that
+		// upload() applies per QSO is not repeated here: closeUploadIDs has
+		// already filtered the sweep down to this logbook's callsign.
+		if m.deps.ShouldUpload != nil && !m.deps.ShouldUpload(ServiceClublog, id) {
+			continue
+		}
+		// Club Log takes the logbook callsign as its own form field, so the ADIF
+		// keeps the QSO's own station call (no override) — as in upload().
+		rec, ok := m.deps.BuildADIF(id, "")
+		if !ok {
+			continue
+		}
+		records = append(records, rec)
+		kept = append(kept, id)
+		if len(records) >= clublogBatchChunk {
+			send()
+		}
+	}
+	send()
+	return uploaded
+}
+
 // flushLoTWBatch signs+uploads all queued LoTW QSOs in one TQSL run, then
 // stamps each as uploaded on success.
 func (m *Manager) flushLoTWBatch(ids []int64, cfg ServiceConfig) int {