feat: Winkeyer

internal/adif/charcount_test.go

@@ -0,0 +1,44 @@
+package adif
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// Loggers (notably Log4OM's UDP/exported ADIF) sometimes declare a field
+// length as the CHARACTER count instead of the UTF-8 byte count, truncating
+// multibyte values mid-rune. The parser must recover the full value.
+func TestCharCountLengthRepair(t *testing.T) {
+	cases := []struct{ name, wantQTH, wantName, adi string }{
+		{
+			name:     "latin",
+			wantQTH:  "Tóalmás", // 7 chars / 9 bytes, declared 7
+			wantName: "Laci Budai",
+			adi:      "<EOH>\n<CALL:5>HA5XY<QTH:7>Tóalmás<NAME:10>Laci Budai<EOR>\n",
+		},
+		{
+			name:     "cyrillic",
+			wantQTH:  "Дзержинск",        // 9 chars / 18 bytes, declared 9
+			wantName: "Александр Чайка",  // 15 chars / 29 bytes, declared 15
+			adi:      "<EOH>\n<CALL:6>UA3TFS<NAME:15>Александр Чайка<QTH:9>Дзержинск<EOR>\n",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			var got Record
+			if err := Parse(strings.NewReader(c.adi), func(r Record) error { got = r; return nil }); err != nil {
+				t.Fatalf("parse: %v", err)
+			}
+			if got["qth"] != c.wantQTH {
+				t.Errorf("qth = %q, want %q", got["qth"], c.wantQTH)
+			}
+			if got["name"] != c.wantName {
+				t.Errorf("name = %q, want %q", got["name"], c.wantName)
+			}
+			if !utf8.ValidString(got["name"]) || !utf8.ValidString(got["qth"]) {
+				t.Errorf("result not valid UTF-8: name=%q qth=%q", got["name"], got["qth"])
+			}
+		})
+	}
+}

internal/adif/export.go

@@ -27,6 +27,13 @@ type Exporter struct {
 	// AppName / AppVersion populate the ADIF header comments. Optional.
 	AppName    string
 	AppVersion string
+
+	// IncludeAppFields controls whether application-specific fields (ADIF
+	// "APP_<programid>_<name>" tags, e.g. Log4OM's APP_LOG4OM_* or our own
+	// OpsLog extensions) are written. Leave false for a clean standard-ADIF
+	// export destined for another logger; set true for a full OpsLog→OpsLog
+	// round-trip that preserves everything.
+	IncludeAppFields bool
 }
 
 // ExportFile creates path (overwriting if it exists) and writes every QSO.
@@ -70,7 +77,7 @@ func (e *Exporter) Export(ctx context.Context, w io.Writer) (int, error) {
 
 	count := 0
 	err := e.Repo.IterateAll(ctx, func(q qso.QSO) error {
-		writeRecord(bw, q)
+		writeRecord(bw, q, e.IncludeAppFields)
 		count++
 		return nil
 	})
@@ -84,7 +91,8 @@ func (e *Exporter) Export(ctx context.Context, w io.Writer) (int, error) {
 func SingleRecordADIF(q qso.QSO) string {
 	var b strings.Builder
 	bw := bufio.NewWriter(&b)
-	writeRecord(bw, q)
+	// Uploads target other services — keep it standard (no app-specific tags).
+	writeRecord(bw, q, false)
 	bw.Flush()
 	return b.String()
 }
@@ -93,7 +101,7 @@ func SingleRecordADIF(q qso.QSO) string {
 // Empty fields are omitted. MODE/SUBMODE are massaged so a "promoted"
 // mode (e.g. FT4 stored without a parent) is exported as the canonical
 // pair MODE=MFSK SUBMODE=FT4 — round-trips cleanly with strict loggers.
-func writeRecord(bw *bufio.Writer, q qso.QSO) {
+func writeRecord(bw *bufio.Writer, q qso.QSO, includeApp bool) {
 	// --- Core ---
 	writeField(bw, "CALL", q.Callsign)
 
@@ -218,8 +226,15 @@ func writeRecord(bw *bufio.Writer, q qso.QSO) {
 	writeField(bw, "NOTES", q.Notes)
 
 	// --- Extras (unpromoted ADIF fields preserved verbatim) ---
+	// In standard mode we drop application-specific tags (APP_*) so the file
+	// stays portable to other loggers; in full mode they're kept for a
+	// lossless OpsLog round-trip.
 	for k, v := range q.Extras {
-		writeField(bw, strings.ToUpper(k), v)
+		tag := strings.ToUpper(k)
+		if !includeApp && strings.HasPrefix(tag, "APP_") {
+			continue
+		}
+		writeField(bw, tag, v)
 	}
 
 	bw.WriteString("<EOR>\n")

internal/adif/import.go

@@ -20,6 +20,7 @@ import (
 type ImportResult struct {
 	Total            int      `json:"total"`             // records found in the file
 	Imported         int      `json:"imported"`          // successfully inserted
+	Updated          int      `json:"updated"`           // existing QSOs refreshed (update-duplicates mode)
 	Skipped          int      `json:"skipped"`           // dropped (missing required fields)
 	Duplicates       int      `json:"duplicates"`        // matched an existing or earlier-in-file QSO
 	DuplicateSamples []string `json:"duplicate_samples"` // up to maxDuplicateSamples human-readable IDs of skipped duplicates
@@ -36,6 +37,19 @@ type Importer struct {
 	Repo           *qso.Repo
 	BatchSize      int  // 0 → 500
 	SkipDuplicates bool // when true, records matching an existing or earlier-in-file QSO are skipped; otherwise all are inserted
+	// UpdateDuplicates, when true, takes precedence over SkipDuplicates:
+	// a record matching an existing QSO MERGES its non-empty fields onto
+	// that QSO (refreshes QSL/confirmation statuses on re-sync) instead of
+	// being skipped or re-inserted.
+	UpdateDuplicates bool
+	// Enrich, when set, is called on each parsed QSO before dedup/insert.
+	// Used to recompute country / zones from cty.dat so a bad COUNTRY in the
+	// source file (common with contest loggers) is corrected on the way in.
+	Enrich func(*qso.QSO)
+	// OnProgress, when set, is called periodically with (processed, total)
+	// record counts so the UI can show a progress bar. total is an estimate
+	// from counting <EOR> tags up front.
+	OnProgress func(processed, total int)
 }
 
 // ImportFile reads the file at path and imports it into the repo. The
@@ -62,6 +76,14 @@ func (im *Importer) ImportFile(ctx context.Context, path string) (ImportResult,
 // Windows-1252 (É is one byte 0xC9). Pre-decoding to UTF-8 would make É
 // two bytes, and the parser reading 7 bytes after the tag would chop the
 // É in half → "YAOUND" + an orphan 0xC3 byte → "YAOUND�" after JSON.
+// ValueDecoderFor returns the per-field byte decoder appropriate for a raw
+// ADIF payload: identity when it's valid UTF-8, otherwise a Windows-1252
+// decoder. Exposed so non-file ingest paths (UDP auto-log from Log4OM /
+// JTAlert) transcode accented NAME/QTH fields the same way file import does.
+func ValueDecoderFor(data []byte) func([]byte) string {
+	return pickValueDecoder(data)
+}
+
 func pickValueDecoder(data []byte) func([]byte) string {
 	if utf8.Valid(data) {
 		return nil // identity
@@ -97,6 +119,15 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 	res := ImportResult{}
 	batch := make([]qso.QSO, 0, im.BatchSize)
 
+	// Up-front record-count estimate (count <EOR> tags, case-insensitive) so
+	// the UI progress bar has a denominator. Cheap single scan.
+	total := countEOR(data)
+	reportProgress := func(force bool) {
+		if im.OnProgress != nil && (force || res.Total%200 == 0) {
+			im.OnProgress(res.Total, total)
+		}
+	}
+
 	// One upfront query for every existing dedup key — cheaper than N
 	// per-record EXISTS calls. The same map gets new keys appended as we
 	// import so duplicates inside the file are caught too. Loaded
@@ -107,6 +138,16 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 		return res, fmt.Errorf("load dedupe keys: %w", err)
 	}
 
+	// Update-duplicates mode needs the existing row's ID per key so it can
+	// fetch, merge and write it back. Loaded only when needed (extra query).
+	var keyIDs map[string]int64
+	if im.UpdateDuplicates {
+		keyIDs, err = im.Repo.DedupeKeyIDs(ctx)
+		if err != nil {
+			return res, fmt.Errorf("load dedupe ids: %w", err)
+		}
+	}
+
 	flush := func() error {
 		if len(batch) == 0 {
 			return nil
@@ -119,6 +160,7 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 
 	err = ParseWithDecoder(bytes.NewReader(data), decode, func(rec Record) error {
 		res.Total++
+		reportProgress(false)
 		q, ok := recordToQSO(rec)
 		if !ok {
 			res.Skipped++
@@ -128,6 +170,9 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 			}
 			return nil
 		}
+		if im.Enrich != nil {
+			im.Enrich(&q)
+		}
 		key := qso.DedupeKey(q.Callsign, q.QSODate.UTC().Format("2006-01-02T15:04"), q.Band, q.Mode)
 		if _, dup := seen[key]; dup {
 			res.Duplicates++
@@ -138,6 +183,28 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 						q.QSODate.UTC().Format("2006-01-02 15:04"),
 						q.Band, q.Mode))
 			}
+			if im.UpdateDuplicates {
+				if id, ok := keyIDs[key]; ok {
+					existing, gerr := im.Repo.GetByID(ctx, id)
+					if gerr != nil {
+						if len(res.Errors) < maxErrors {
+							res.Errors = append(res.Errors,
+								fmt.Sprintf("record %d (%s): load existing: %v", res.Total, q.Callsign, gerr))
+						}
+						return nil
+					}
+					qso.MergeNonZero(&existing, q)
+					if uerr := im.Repo.Update(ctx, existing); uerr != nil {
+						if len(res.Errors) < maxErrors {
+							res.Errors = append(res.Errors,
+								fmt.Sprintf("record %d (%s): update: %v", res.Total, q.Callsign, uerr))
+						}
+						return nil
+					}
+					res.Updated++
+				}
+				return nil
+			}
 			if im.SkipDuplicates {
 				return nil
 			}
@@ -159,9 +226,28 @@ func (im *Importer) importBytes(ctx context.Context, data []byte) (ImportResult,
 	if err := flush(); err != nil {
 		return res, err
 	}
+	reportProgress(true) // final 100%
 	return res, nil
 }
 
+// countEOR estimates the record count by counting case-insensitive <EOR>
+// tags. Used only to give the import progress bar a denominator.
+func countEOR(data []byte) int {
+	n := 0
+	for i := 0; i+4 <= len(data); i++ {
+		if data[i] != '<' {
+			continue
+		}
+		if (data[i+1] == 'e' || data[i+1] == 'E') &&
+			(data[i+2] == 'o' || data[i+2] == 'O') &&
+			(data[i+3] == 'r' || data[i+3] == 'R') &&
+			(i+4 < len(data) && (data[i+4] == '>' || data[i+4] == ':')) {
+			n++
+		}
+	}
+	return n
+}
+
 // adifPromoted lists every lowercase ADIF tag that maps to a promoted column.
 // Anything not in this set ends up in Extras.
 var adifPromoted = stringSet(
@@ -178,7 +264,7 @@ var adifPromoted = stringSet(
 	"age", "lat", "lon", "rig", "ant",
 	// QSL
 	"qsl_sent", "qsl_rcvd",
-	"qslsdate", "qslrdate", "qsl_via", "qslmsg", "qslmsg_rcvd",
+	"qslsdate", "qslrdate", "qsl_via", "qsl_sent_via", "qsl_rcvd_via", "qslmsg", "qslmsg_rcvd",
 	"lotw_qsl_sent", "lotw_qsl_rcvd", "lotw_qslsdate", "lotw_qslrdate",
 	"eqsl_qsl_sent", "eqsl_qsl_rcvd", "eqsl_qslsdate", "eqsl_qslrdate",
 	"clublog_qso_upload_date", "clublog_qso_upload_status",
@@ -300,6 +386,9 @@ func recordToQSO(rec Record) (qso.QSO, bool) {
 	q.QSLSentDate = rec["qslsdate"]
 	q.QSLRcvdDate = rec["qslrdate"]
 	q.QSLVia = rec["qsl_via"]
+	if q.QSLVia == "" { // many loggers (Log4OM) write QSL_SENT_VIA instead
+		q.QSLVia = rec["qsl_sent_via"]
+	}
 	q.QSLMsg = rec["qslmsg"]
 	q.QSLMsgRcvd = rec["qslmsg_rcvd"]
 	q.LOTWSent = rec["lotw_qsl_sent"]

internal/adif/parser.go

@@ -16,6 +16,7 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 )
 
 // Record is a single ADIF record. Keys are lowercased field names.
@@ -83,6 +84,17 @@ func parseWith(r io.Reader, decodeValue func([]byte) string, fn func(Record) err
 			if _, err := io.ReadFull(br, val); err != nil {
 				return fmt.Errorf("read field %s: %w", name, err)
 			}
+			// Repair character-count lengths. The ADIF spec says LENGTH is a
+			// byte count, but some loggers (notably Log4OM's UDP "ADIF
+			// message") write the CHARACTER count instead. For UTF-8 values
+			// with accented chars that truncates mid-rune — e.g. "<QTH:7>
+			// Tóalmás" is 9 bytes but says 7, leaving an orphan byte that
+			// renders as "Tóalm�". When we're in UTF-8 mode (no Windows-1252
+			// decoder) and the naive byte read isn't valid UTF-8, keep reading
+			// until the value holds `length` whole runes (or the next tag).
+			if decodeValue == nil && !utf8.Valid(val) {
+				val = extendToRunes(br, val, length)
+			}
 			if headerDone && name != "" {
 				if decodeValue != nil {
 					rec[name] = decodeValue(val)
@@ -94,6 +106,37 @@ func parseWith(r io.Reader, decodeValue func([]byte) string, fn func(Record) err
 	}
 }
 
+// extendToRunes recovers a value whose declared length was a character count
+// rather than a byte count. `have` holds the first `wantRunes` BYTES of the
+// value, which turned out to be invalid UTF-8 (a multibyte rune was cut). We
+// append bytes from br until the value holds `wantRunes` complete runes — or
+// until the next '<' (start of the following tag) / EOF, so we never cross
+// into another field. Capped so a genuinely-corrupt value can't run away.
+func extendToRunes(br *bufio.Reader, have []byte, wantRunes int) []byte {
+	const maxExtra = 8 // at most ~4 extra bytes/rune for the few cut runes
+	limit := len(have) + maxExtra*wantRunes + maxExtra
+	for len(have) < limit {
+		// Stop only when the value is complete UTF-8 (no partial trailing
+		// rune) AND holds enough runes. Checking utf8.RuneCount alone is a
+		// trap: a trailing orphan lead byte (e.g. the D0 of a cut Cyrillic
+		// "а") counts as one rune, so the loop would stop one continuation
+		// byte short → "Чайк�". Requiring utf8.Valid forces us to read it.
+		if utf8.Valid(have) && utf8.RuneCount(have) >= wantRunes {
+			break
+		}
+		b, err := br.ReadByte()
+		if err != nil {
+			break
+		}
+		if b == '<' {
+			_ = br.UnreadByte() // belongs to the next tag — leave it
+			break
+		}
+		have = append(have, b)
+	}
+	return have
+}
+
 // parseSpec splits "callsign:5", "callsign:5:S" or "eor" into name and length.
 // name is lowercased; length is 0 for control tags or when missing.
 func parseSpec(spec string) (name string, length int) {