update

internal/adif/import.go

@@ -13,18 +13,24 @@ import (
 
 // ImportResult summarises an ADIF import for the UI.
 type ImportResult struct {
-	Total    int      `json:"total"`    // records found in the file
-	Imported int      `json:"imported"` // successfully inserted
-	Skipped  int      `json:"skipped"`  // dropped (missing required fields, etc.)
-	Errors   []string `json:"errors"`   // up to maxErrors error messages
+	Total            int      `json:"total"`             // records found in the file
+	Imported         int      `json:"imported"`          // successfully inserted
+	Skipped          int      `json:"skipped"`           // dropped (missing required fields)
+	Duplicates       int      `json:"duplicates"`        // matched an existing or earlier-in-file QSO
+	DuplicateSamples []string `json:"duplicate_samples"` // up to maxDuplicateSamples human-readable IDs of skipped duplicates
+	Errors           []string `json:"errors"`            // up to maxErrors error messages
 }
 
-const maxErrors = 50
+const (
+	maxErrors           = 50
+	maxDuplicateSamples = 200 // cap memory + JSON payload size
+)
 
 // Importer streams an ADI file into a QSO repository.
 type Importer struct {
-	Repo      *qso.Repo
-	BatchSize int // 0 → 500
+	Repo           *qso.Repo
+	BatchSize      int  // 0 → 500
+	SkipDuplicates bool // when true, records matching an existing or earlier-in-file QSO are skipped; otherwise all are inserted
 }
 
 // ImportFile opens the file at path and imports it into the repo.
@@ -47,6 +53,16 @@ func (im *Importer) Import(ctx context.Context, r interface {
 	res := ImportResult{}
 	batch := make([]qso.QSO, 0, im.BatchSize)
 
+	// One upfront query for every existing dedup key — cheaper than N
+	// per-record EXISTS calls. The same map gets new keys appended as we
+	// import so duplicates inside the file are caught too. Loaded
+	// unconditionally so the "duplicates" count is meaningful even when
+	// SkipDuplicates is off (the user still wants to know how many).
+	seen, err := im.Repo.ExistingDedupeKeys(ctx)
+	if err != nil {
+		return res, fmt.Errorf("load dedupe keys: %w", err)
+	}
+
 	flush := func() error {
 		if len(batch) == 0 {
 			return nil
@@ -57,7 +73,7 @@ func (im *Importer) Import(ctx context.Context, r interface {
 		return err
 	}
 
-	err := Parse(r, func(rec Record) error {
+	err = Parse(r, func(rec Record) error {
 		res.Total++
 		q, ok := recordToQSO(rec)
 		if !ok {
@@ -68,6 +84,24 @@ func (im *Importer) Import(ctx context.Context, r interface {
 			}
 			return nil
 		}
+		key := qso.DedupeKey(q.Callsign, q.QSODate.UTC().Format("2006-01-02T15:04"), q.Band, q.Mode)
+		if _, dup := seen[key]; dup {
+			res.Duplicates++
+			if len(res.DuplicateSamples) < maxDuplicateSamples {
+				res.DuplicateSamples = append(res.DuplicateSamples,
+					fmt.Sprintf("%s · %s · %s · %s",
+						q.Callsign,
+						q.QSODate.UTC().Format("2006-01-02 15:04"),
+						q.Band, q.Mode))
+			}
+			if im.SkipDuplicates {
+				return nil
+			}
+			// Fall through: insert anyway. Don't add to seen[] — keeps
+			// the duplicate count meaningful if the same row appears 3x.
+		} else {
+			seen[key] = struct{}{}
+		}
 		batch = append(batch, q)
 		if len(batch) >= im.BatchSize {
 			return flush()
@@ -137,10 +171,21 @@ func recordToQSO(rec Record) (qso.QSO, bool) {
 	}
 	band := strings.ToLower(strings.TrimSpace(rec["band"]))
 	mode := strings.ToUpper(strings.TrimSpace(rec["mode"]))
+	submode := strings.ToUpper(strings.TrimSpace(rec["submode"]))
 	date := parseDateTime(rec["qso_date"], rec["time_on"])
 	if date.IsZero() || band == "" || mode == "" {
 		return qso.QSO{}, false
 	}
+	// ADIF promotes specific digital flavours into the SUBMODE field with
+	// a generic parent in MODE (e.g. MODE=MFSK SUBMODE=FT4). Loggers
+	// uniformly display the submode in that case — promote it so all
+	// downstream code (entry strip, worked-before, exports) sees "FT4"
+	// rather than "MFSK". SSB+USB/LSB and RTTY+JOH stay on the parent
+	// because the parent IS the displayed mode there.
+	if submode != "" && submodeSubsumesParent(submode) {
+		mode = submode
+		submode = "" // redundant once promoted
+	}
 
 	q := qso.QSO{
 		Callsign:   call,
@@ -149,7 +194,7 @@ func recordToQSO(rec Record) (qso.QSO, bool) {
 		Band:       band,
 		BandRX:     strings.ToLower(rec["band_rx"]),
 		Mode:       mode,
-		Submode:    strings.ToUpper(rec["submode"]),
+		Submode:    submode,
 	}
 	if hz, ok := parseFreqHz(rec["freq"]); ok {
 		q.FreqHz = &hz
@@ -362,3 +407,34 @@ func parseFloat(s string) (float64, bool) {
 	}
 	return v, true
 }
+
+// promotableSubmodes are SUBMODE values that uniquely identify the mode
+// every logger displays — the ADIF parent (MFSK, DATA, PSK, JT…) is
+// generic and unhelpful to show. SSB's USB/LSB and RTTY's JOH are NOT
+// here because operators want to see "SSB" / "RTTY", not the sideband
+// or RTTY variant.
+var promotableSubmodes = map[string]bool{
+	// MFSK family (modern FT/Q65/JS8/MSK144 live here)
+	"FT2": true, "FT4": true, "FT8": true, "JS8": true, "MSK144": true, "ISCAT": true,
+	"Q65": true, "FST4": true, "FST4W": true,
+	"MFSK16": true, "MFSK32": true, "MFSK64": true, "MFSK128": true,
+	"OLIVIA": true,
+	// JT family (some loggers still parent these under DATA)
+	"JT65": true, "JT9": true, "JT4": true, "JT6M": true, "JT44": true, "T10": true,
+	// PSK family
+	"PSK31": true, "PSK63": true, "PSK125": true, "PSK250": true, "PSK500": true,
+	"QPSK31": true, "QPSK63": true, "QPSK125": true, "QPSK250": true, "QPSK500": true,
+	// THOR / DOMINO / HELL
+	"THOR4": true, "THOR8": true, "THOR16": true, "THOR32": true,
+	"DOMINOF": true, "DOMINOEX": true,
+	"HELL80": true, "FMHELL": true,
+	// DIGITALVOICE variants
+	"FREEDV": true,
+	// VARA family — Log4OM parents these under DYNAMIC. VarAC chat uses
+	// "VARA HF" (with the space) so we match both spellings.
+	"VARA": true, "VARA HF": true, "VARA FM": true, "VARAC": true,
+}
+
+func submodeSubsumesParent(submode string) bool {
+	return promotableSubmodes[submode]
+}