feat: Winkeyer

internal/adif/import.go

@@ -20,6 +20,7 @@ import (
 type ImportResult struct {
 	Total            int      `json:"total"`             // records found in the file
 	Imported         int      `json:"imported"`          // successfully inserted
+	Updated          int      `json:"updated"`           // existing QSOs refreshed (update-duplicates mode)
 	Skipped          int      `json:"skipped"`           // dropped (missing required fields)
 	Duplicates       int      `json:"duplicates"`        // matched an existing or earlier-in-file QSO
 	DuplicateSamples []string `json:"duplicate_samples"` // up to maxDuplicateSamples human-readable IDs of skipped duplicates
@@ -36,6 +37,19 @@ type Importer struct {
 	Repo           *qso.Repo
 	BatchSize      int  // 0 → 500
 	SkipDuplicates bool // when true, records matching an existing or earlier-in-file QSO are skipped; otherwise all are inserted
+	// UpdateDuplicates, when true, takes precedence over SkipDuplicates:
+	// a record matching an existing QSO MERGES its non-empty fields onto
+	// that QSO (refreshes QSL/confirmation statuses on re-sync) instead of
+	// being skipped or re-inserted.
+	UpdateDuplicates bool
+	// Enrich, when set, is called on each parsed QSO before dedup/insert.
+	// Used to recompute country / zones from cty.dat so a bad COUNTRY in the
+	// source file (common with contest loggers) is corrected on the way in.
+	Enrich func(*qso.QSO)
+	// OnProgress, when set, is called periodically with (processed, total)
+	// record counts so the UI can show a progress bar. total is an estimate
+	// from counting <EOR> tags up front.
+	OnProgress func(processed, total int)
 }
 
 // ImportFile reads the file at path and imports it into the repo. The
@@ -62,6 +76,14 @@ func (im *Importer) ImportFile(ctx context.Context, path string) (ImportResult,
 // Windows-1252 (É is one byte 0xC9). Pre-decoding to UTF-8 would make É
 // two bytes, and the parser reading 7 bytes after the tag would chop the
 // É in half → "YAOUND" + an orphan 0xC3 byte → "YAOUND�" after JSON.
+// ValueDecoderFor returns the per-field byte decoder appropriate for a raw
+// ADIF payload: identity when it's valid UTF-8, otherwise a Windows-1252
+// decoder. Exposed so non-file ingest paths (UDP auto-log from Log4OM /
+// JTAlert) transcode accented NAME/QTH fields the same way file import does.
+func ValueDecoderFor(data []byte) func([]byte) string {
+	return pickValueDecoder(data)
+}
+
 func pickValueDecoder(data []byte) func([]byte) string {
 	if utf8.Valid(data) {
 		return nil // identity
@@ -97,6 +119,15 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 	res := ImportResult{}
 	batch := make([]qso.QSO, 0, im.BatchSize)
 
+	// Up-front record-count estimate (count <EOR> tags, case-insensitive) so
+	// the UI progress bar has a denominator. Cheap single scan.
+	total := countEOR(data)
+	reportProgress := func(force bool) {
+		if im.OnProgress != nil && (force || res.Total%200 == 0) {
+			im.OnProgress(res.Total, total)
+		}
+	}
+
 	// One upfront query for every existing dedup key — cheaper than N
 	// per-record EXISTS calls. The same map gets new keys appended as we
 	// import so duplicates inside the file are caught too. Loaded
@@ -107,6 +138,16 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 		return res, fmt.Errorf("load dedupe keys: %w", err)
 	}
 
+	// Update-duplicates mode needs the existing row's ID per key so it can
+	// fetch, merge and write it back. Loaded only when needed (extra query).
+	var keyIDs map[string]int64
+	if im.UpdateDuplicates {
+		keyIDs, err = im.Repo.DedupeKeyIDs(ctx)
+		if err != nil {
+			return res, fmt.Errorf("load dedupe ids: %w", err)
+		}
+	}
+
 	flush := func() error {
 		if len(batch) == 0 {
 			return nil
@@ -119,6 +160,7 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 
 	err = ParseWithDecoder(bytes.NewReader(data), decode, func(rec Record) error {
 		res.Total++
+		reportProgress(false)
 		q, ok := recordToQSO(rec)
 		if !ok {
 			res.Skipped++
@@ -128,6 +170,9 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 			}
 			return nil
 		}
+		if im.Enrich != nil {
+			im.Enrich(&q)
+		}
 		key := qso.DedupeKey(q.Callsign, q.QSODate.UTC().Format("2006-01-02T15:04"), q.Band, q.Mode)
 		if _, dup := seen[key]; dup {
 			res.Duplicates++
@@ -138,6 +183,28 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 						q.QSODate.UTC().Format("2006-01-02 15:04"),
 						q.Band, q.Mode))
 			}
+			if im.UpdateDuplicates {
+				if id, ok := keyIDs[key]; ok {
+					existing, gerr := im.Repo.GetByID(ctx, id)
+					if gerr != nil {
+						if len(res.Errors) < maxErrors {
+							res.Errors = append(res.Errors,
+								fmt.Sprintf("record %d (%s): load existing: %v", res.Total, q.Callsign, gerr))
+						}
+						return nil
+					}
+					qso.MergeNonZero(&existing, q)
+					if uerr := im.Repo.Update(ctx, existing); uerr != nil {
+						if len(res.Errors) < maxErrors {
+							res.Errors = append(res.Errors,
+								fmt.Sprintf("record %d (%s): update: %v", res.Total, q.Callsign, uerr))
+						}
+						return nil
+					}
+					res.Updated++
+				}
+				return nil
+			}
 			if im.SkipDuplicates {
 				return nil
 			}
@@ -159,9 +226,28 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 	if err := flush(); err != nil {
 		return res, err
 	}
+	reportProgress(true) // final 100%
 	return res, nil
 }
 
+// countEOR estimates the record count by counting case-insensitive <EOR>
+// tags. Used only to give the import progress bar a denominator.
+func countEOR(data []byte) int {
+	n := 0
+	for i := 0; i+4 <= len(data); i++ {
+		if data[i] != '<' {
+			continue
+		}
+		if (data[i+1] == 'e' || data[i+1] == 'E') &&
+			(data[i+2] == 'o' || data[i+2] == 'O') &&
+			(data[i+3] == 'r' || data[i+3] == 'R') &&
+			(i+4 < len(data) && (data[i+4] == '>' || data[i+4] == ':')) {
+			n++
+		}
+	}
+	return n
+}
+
 // adifPromoted lists every lowercase ADIF tag that maps to a promoted column.
 // Anything not in this set ends up in Extras.
 var adifPromoted = stringSet(
@@ -178,7 +264,7 @@ var adifPromoted = stringSet(
 	"age", "lat", "lon", "rig", "ant",
 	// QSL
 	"qsl_sent", "qsl_rcvd",
-	"qslsdate", "qslrdate", "qsl_via", "qslmsg", "qslmsg_rcvd",
+	"qslsdate", "qslrdate", "qsl_via", "qsl_sent_via", "qsl_rcvd_via", "qslmsg", "qslmsg_rcvd",
 	"lotw_qsl_sent", "lotw_qsl_rcvd", "lotw_qslsdate", "lotw_qslrdate",
 	"eqsl_qsl_sent", "eqsl_qsl_rcvd", "eqsl_qslsdate", "eqsl_qslrdate",
 	"clublog_qso_upload_date", "clublog_qso_upload_status",
@@ -300,6 +386,9 @@ func recordToQSO(rec Record) (qso.QSO, bool) {
 	q.QSLSentDate = rec["qslsdate"]
 	q.QSLRcvdDate = rec["qslrdate"]
 	q.QSLVia = rec["qsl_via"]
+	if q.QSLVia == "" { // many loggers (Log4OM) write QSL_SENT_VIA instead
+		q.QSLVia = rec["qsl_sent_via"]
+	}
 	q.QSLMsg = rec["qslmsg"]
 	q.QSLMsgRcvd = rec["qslmsg_rcvd"]
 	q.LOTWSent = rec["lotw_qsl_sent"]