OpsLog/hamlogcfm.go

package main

// Reading confirmations back from a HAMLOG.online ADIF export.
//
// Their agent protocol only uploads — it has no verb for asking what has been
// confirmed — so the return path is a file: their site exports an ADIF, and the
// operator feeds it back here.
//
// That file must NOT be imported the ordinary way. A plain import matches a
// record to a local QSO on callsign + UTC MINUTE + band + mode, and their
// export is rebuilt from their own database: a minute of rounding, SSB where
// the log says USB, and the key no longer matches. "Update duplicates" then
// does what it is told — no duplicate found, so it inserts — and a few hundred
// copies of contacts the operator already had land in the log. That is exactly
// what happened once, and it is why this path exists instead.
//
// So: match only, never insert. Confirmations are stamped onto the QSOs already
// in the log, and anything that cannot be matched is REPORTED rather than
// added, because an unmatched confirmation is a question about the log (a
// minute off, a portable call) and not a contact to create.

import (
	"context"
	"fmt"
	"os"
	"strings"
	"time"

	wruntime "github.com/wailsapp/wails/v2/pkg/runtime"

	"hamlog/internal/adif"
	"hamlog/internal/applog"
	"hamlog/internal/award"
	"hamlog/internal/qso"
)

// hamlogUnmatchedMax bounds what is kept for export. A whole log's worth of
// unmatched records means the file belongs to another station, not that the
// operator wants 50 000 of them written back out.
const hamlogUnmatchedMax = 20000

// HamlogCfmResult is what the import did.
type HamlogCfmResult struct {
	Total     int `json:"total"`     // records read from the file
	Confirmed int `json:"confirmed"` // records carrying HAMLOG's confirmation flag
	Matched   int `json:"matched"`   // local QSOs stamped
	ByClass   int `json:"by_class"`  // matched on mode CLASS rather than exact mode
	Unmatched int `json:"unmatched"` // confirmations with no local QSO
	// Samples names a few unmatched contacts, so "12 unmatched" can be looked
	// into rather than merely worried about. The full list is kept for export —
	// see ExportHamlogUnmatched — because 395 of them is not a sample-sized
	// problem: it is a list to work through.
	Samples []string `json:"samples"`
}

// hamlogCfmSamples caps the reported list — enough to see the pattern, not so
// many that the dialog becomes a log file.
const hamlogCfmSamples = 30

// ImportHamlogConfirmations stamps the confirmations from a HAMLOG.online ADIF
// export onto the matching local QSOs. It inserts nothing.
func (a *App) ImportHamlogConfirmations(path string) (HamlogCfmResult, error) {
	var res HamlogCfmResult
	if a.qso == nil {
		return res, fmt.Errorf("db not initialized")
	}
	if strings.TrimSpace(path) == "" {
		return res, fmt.Errorf("empty path")
	}
	f, err := os.Open(path)
	if err != nil {
		return res, err
	}
	defer f.Close()

	ctx := a.ctx
	if ctx == nil {
		ctx = context.Background()
	}
	// The same two indexes the LoTW download uses: the exact key, then the
	// mode-CLASS key for the contacts whose mode was written differently at the
	// other end (FT8 exported as DATA, SSB where the log says USB).
	keyIDs, err := a.qso.DedupeKeyIDs(ctx)
	if err != nil {
		return res, fmt.Errorf("read local log: %w", err)
	}
	classIDs, _ := a.qso.DedupeClassKeyIDs(ctx)

	emit := func(line string) {
		if a.ctx != nil {
			wruntime.EventsEmit(a.ctx, "qslmgr:log", line)
		}
	}
	emit("Reading " + path + "…")

	// What already counts towards an award, so each confirmation can be flagged
	// NEW. LoTW and paper QSL are the two award-valid sources; a HAMLOG
	// confirmation is "new" when it lands on a slot neither of them holds — which
	// is the only sense in which it changes anything.
	sets, _ := a.qso.ConfirmedSlots(ctx, []string{"lotw_rcvd", "qsl_rcvd"})
	var items []ConfirmationItem
	var unmatched []qso.QSO

	perr := adif.Parse(f, func(rec adif.Record) error {
		q, ok := adif.RecordToQSO(rec)
		if !ok {
			return nil
		}
		res.Total++
		// Only the records they actually confirmed. The export carries the whole
		// log, and stamping "confirmed" on all of it would turn every uploaded
		// contact into a confirmed one.
		if !hamlogRecordConfirmed(rec) {
			return nil
		}
		res.Confirmed++

		minute := q.QSODate.UTC().Format("2006-01-02T15:04")
		id, found := keyIDs[qso.DedupeKey(q.Callsign, minute, q.Band, q.Mode)]
		if !found {
			// id 0 means the class key is ambiguous — several local QSOs share
			// it — and guessing between them would stamp the wrong one.
			if cid, ok := classIDs[qso.DedupeClassKey(q.Callsign, minute, q.Band, q.Mode)]; ok && cid != 0 {
				id, found = cid, true
				res.ByClass++
			}
		}
		if !found {
			res.Unmatched++
			if len(res.Samples) < hamlogCfmSamples {
				res.Samples = append(res.Samples, fmt.Sprintf("%s · %s · %s · %s",
					q.Callsign, q.QSODate.UTC().Format("2006-01-02 15:04Z"), q.Band, q.Mode))
			}
			if len(unmatched) < hamlogUnmatchedMax {
				unmatched = append(unmatched, q)
			}
			return nil
		}
		date := hamlogCfmDate(rec)
		if e := a.qso.SetExtra(ctx, id, award.HamlogQSLKey, "Y"); e != nil {
			return nil
		}
		_ = a.qso.SetExtra(ctx, id, hamlogQSLDateKey, date)
		// A confirmed contact is by definition one they hold, so the sent side
		// is true whether or not OpsLog is what uploaded it — a log uploaded
		// from their website would otherwise read "never sent, yet confirmed".
		_ = a.qso.SetExtra(ctx, id, hamlogSentKey, "Y")
		res.Matched++

		// Feed the Results view, the same rows the LoTW download produces: an
		// import that only prints counts leaves the operator with no way to see
		// WHICH contacts were confirmed, which is the reason they ran it.
		a.enrichContactedFromCty(&q) // country/dxcc, for the entity flags
		it := ConfirmationItem{
			Callsign: q.Callsign,
			QSODate:  q.QSODate.UTC().Format(time.RFC3339),
			Band:     q.Band,
			Mode:     q.Mode,
			Country:  q.Country,
		}
		if q.DXCC != nil && *q.DXCC != 0 {
			n := *q.DXCC
			it.NewDXCC = !sets.DXCC[n]
			it.NewBand = !sets.Band[qso.BandKey(n, q.Band)]
			it.NewMode = !sets.Mode[qso.ModeClassKey(n, q.Mode)]
			it.NewSlot = !sets.Slot[qso.SlotClassKey(n, q.Band, q.Mode)]
			// Fold it in, so a repeat inside the same file isn't flagged twice.
			sets.DXCC[n] = true
			sets.Band[qso.BandKey(n, q.Band)] = true
			sets.Mode[qso.ModeClassKey(n, q.Mode)] = true
			sets.Slot[qso.SlotClassKey(n, q.Band, q.Mode)] = true
		}
		items = append(items, it)
		return nil
	})
	if perr != nil {
		return res, perr
	}
	a.invalidateAwardStats() // confirmations move award counts
	// Kept for ExportHamlogUnmatched. Held rather than written now: the operator
	// decides whether a list of 395 is worth a file.
	a.hamlogUnmatchedMu.Lock()
	a.hamlogUnmatched = unmatched
	a.hamlogUnmatchedMu.Unlock()
	if a.ctx != nil {
		wruntime.EventsEmit(a.ctx, "qslmgr:confirmations", items)
	}
	applog.Printf("hamlog cfm import: %d records, %d confirmed, %d matched (%d by mode class), %d unmatched",
		res.Total, res.Confirmed, res.Matched, res.ByClass, res.Unmatched)
	emit(fmt.Sprintf("%d records read, %d confirmed by HAMLOG.online, %d matched in the log (%d by mode class), %d unmatched",
		res.Total, res.Confirmed, res.Matched, res.ByClass, res.Unmatched))
	for _, s := range res.Samples {
		emit("  unmatched: " + s)
	}
	if res.Unmatched > len(res.Samples) {
		emit(fmt.Sprintf("  …and %d more — use \"Export unmatched (ADIF)…\" to get the whole list.",
			res.Unmatched-len(res.Samples)))
	}
	if a.ctx != nil {
		wruntime.EventsEmit(a.ctx, "qslmgr:done", map[string]any{"uploaded": res.Matched, "total": res.Confirmed})
	}
	return res, nil
}

// ExportHamlogUnmatched writes the confirmations the last import could not
// place onto a QSO.
//
// They are the interesting half of the result: each one is a contact HAMLOG
// believes it holds and this log does not agree about — a minute of drift, a
// portable call, a band written differently, or a QSO genuinely missing. A
// count cannot be worked through; a file can be opened, sorted and compared.
//
// Written as ADIF because that is what every other tool reads, and because it
// can be handed straight back to an import once the discrepancies are settled.
func (a *App) ExportHamlogUnmatched(path string) (int, error) {
	a.hamlogUnmatchedMu.Lock()
	rows := a.hamlogUnmatched
	a.hamlogUnmatchedMu.Unlock()
	if len(rows) == 0 {
		return 0, fmt.Errorf("nothing to export: the last import left no unmatched confirmations")
	}
	if strings.TrimSpace(path) == "" {
		return 0, fmt.Errorf("empty path")
	}
	recs := make([]string, 0, len(rows))
	for i := range rows {
		recs = append(recs, adif.FullRecordADIF(rows[i]))
	}
	if err := os.WriteFile(path, []byte(adif.BatchRecordsADIF(recs)), 0o644); err != nil {
		return 0, err
	}
	applog.Printf("hamlog cfm import: exported %d unmatched confirmation(s) to %s", len(rows), path)
	return len(rows), nil
}

// hamlogQSLDateKey stamps WHEN the confirmation was read back. Their export
// carries no confirmation date of its own, so this is the import date — which
// is honest about what it knows, unlike borrowing the QSO date.
const hamlogQSLDateKey = "APP_OPSLOG_HAMLOG_QSL_DATE"

// hamlogRecordConfirmed reads their flag off a raw ADIF record. The primary key
// is the one their own export writes; the alternates are the names OpsLog and
// other tools have used, so a file that went through another logger still reads.
func hamlogRecordConfirmed(rec adif.Record) bool {
	keys := append([]string{award.HamlogQSLKey}, award.HamlogAltKeys()...)
	for _, k := range keys {
		v := strings.ToUpper(strings.TrimSpace(rec[strings.ToLower(k)]))
		if v != "" && v != "N" && v != "NO" {
			return true
		}
	}
	return false
}

// hamlogCfmDate is the date to stamp: the import date, in ADIF form.
func hamlogCfmDate(rec adif.Record) string {
	// If a future export ever carries one, take it rather than today's date.
	for _, k := range []string{"app_hamlog_qso_cfm_date", "qslrdate"} {
		if v := strings.TrimSpace(rec[k]); len(v) == 8 {
			return v
		}
	}
	return time.Now().UTC().Format("20060102")
}