OpsLog/rdacompare.go

package main

// Comparing the two sources of a Russian district, BEFORE deciding which wins.
//
// A QSO can carry an RDA district from two places now:
//
//   - the offline RDA database (internal/rda), compiled by the award's own
//     administrators, which knows where a callsign was on a GIVEN DAY;
//   - the CNTY field of a log downloaded from HAMLOG.online, whose export
//     writes the district there (<CNTY:5>RO-19 on a real record).
//
// Whether that second source is EVIDENCE or merely a COPY is not a matter of
// opinion, and it decides everything downstream: if HAMLOG's value always
// equals the database's, there is nothing to arbitrate and a precedence rule
// would be ceremony around a redundancy. If they genuinely differ on some
// contacts, then two independent records disagree about a QSO that counts for
// an award, and that has to be surfaced rather than silently resolved.
//
// So: measure first. This reports agreements, disagreements, and the contacts
// where only one side has an answer — and it changes nothing in the log.

import (
	"fmt"
	"regexp"
	"strings"

	"hamlog/internal/applog"
	"hamlog/internal/award"
	"hamlog/internal/qso"
	"hamlog/internal/rda"
)

// rdaDistrictRe is the shape of an RDA reference: two letters, a dash, two
// digits (RO-19, KO-05). Used to tell a real district in CNTY from a US county
// name or whatever else a callbook dropped in that field.
var rdaDistrictRe = regexp.MustCompile(`^[A-Z]{2}-[0-9]{2}$`)

// RDAConflict is one contact where the two sources disagree.
type RDAConflict struct {
	QSOID    int64  `json:"qso_id"`
	Callsign string `json:"callsign"`
	Date     string `json:"date"`
	Band     string `json:"band"`
	Mode     string `json:"mode"`
	// FromLog is the district the imported log carries (CNTY).
	FromLog string `json:"from_log"`
	// FromDB is what the offline RDA database answers for that callsign on that
	// day, and Dated says whether it was a dated activity record (a fact) or the
	// callsign's current district (an assumption).
	FromDB string `json:"from_db"`
	Dated  bool   `json:"dated"`
	// Confirmed is whether the QSO is confirmed on HAMLOG — a confirmed contact's
	// CNTY has been through their matching of two logs, which is what makes it
	// worth weighing against the database at all.
	Confirmed bool `json:"confirmed"`
}

// RDACompareResult is the measurement.
type RDACompareResult struct {
	Scanned   int `json:"scanned"`    // Russian QSOs examined
	BothKnown int `json:"both_known"` // a district from each source
	Agree     int `json:"agree"`
	Disagree  int `json:"disagree"`
	OnlyLog   int `json:"only_log"` // CNTY has it, the database does not
	OnlyDB    int `json:"only_db"`  // the database has it, the log does not
	Neither   int `json:"neither"`
	// Conflicts lists the disagreements, newest first, capped so a large log
	// cannot produce a dialog nobody can read. Count is in Disagree regardless.
	Conflicts []RDAConflict `json:"conflicts"`
}

// rdaCompareMax bounds the returned list. The COUNT is always exact; it is the
// enumeration that is capped, because a hundred rows is already more than
// anyone reviews in one sitting.
const rdaCompareMax = 200

// CompareRDASources measures the two sources against each other. Read-only.
func (a *App) CompareRDASources() (RDACompareResult, error) {
	var res RDACompareResult
	if a.qso == nil {
		return res, fmt.Errorf("db not initialized")
	}
	// Said before the work starts, not only after: this reads the WHOLE log, and
	// on a remote MySQL that is seconds of silence during which the only
	// evidence that anything is happening is this line.
	applog.Printf("rda compare: reading the log…")
	rows, err := a.qso.List(a.ctx, qso.ListFilter{Limit: 1_000_000})
	if err != nil {
		applog.Printf("rda compare: reading the log failed: %v", err)
		return res, err
	}
	for i := range rows {
		q := &rows[i]
		// The same three entities the award covers — see BackfillRDA.
		if q.DXCC == nil {
			continue
		}
		switch *q.DXCC {
		case 15, 54, 126:
		default:
			continue
		}
		res.Scanned++

		fromLog := strings.ToUpper(strings.TrimSpace(q.County))
		if !rdaDistrictRe.MatchString(fromLog) {
			fromLog = "" // a county name, a stray value: not a district
		}
		var fromDB string
		var dated bool
		if m, ok := rda.Lookup(q.Callsign, q.QSODate); ok {
			fromDB, dated = m.District, m.Dated
		}

		switch {
		case fromLog == "" && fromDB == "":
			res.Neither++
		case fromLog == "":
			res.OnlyDB++
		case fromDB == "":
			res.OnlyLog++
		default:
			res.BothKnown++
			if fromLog == fromDB {
				res.Agree++
				continue
			}
			res.Disagree++
			if len(res.Conflicts) < rdaCompareMax {
				res.Conflicts = append(res.Conflicts, RDAConflict{
					QSOID: q.ID, Callsign: q.Callsign,
					Date: q.QSODate.UTC().Format("2006-01-02"),
					Band: q.Band, Mode: q.Mode,
					FromLog: fromLog, FromDB: fromDB, Dated: dated,
					Confirmed: q.Extras != nil && isHamlogConfirmed(q.Extras[award.HamlogQSLKey]),
				})
			}
		}
	}
	applog.Printf("rda compare: %d Russian QSOs — both %d (agree %d, disagree %d), only-log %d, only-db %d, neither %d",
		res.Scanned, res.BothKnown, res.Agree, res.Disagree, res.OnlyLog, res.OnlyDB, res.Neither)
	return res, nil
}

// isHamlogConfirmed reads their Y — anything that is not an explicit no counts,
// the same rule the award engine applies.
func isHamlogConfirmed(v string) bool {
	v = strings.ToUpper(strings.TrimSpace(v))
	return v != "" && v != "N" && v != "NO"
}

// RDAChoice is one arbitrated contact: which district the operator decided is
// right for that QSO.
type RDAChoice struct {
	QSOID    int64  `json:"qso_id"`
	District string `json:"district"`
}

// RDAApplyResult reports what an arbitration did.
type RDAApplyResult struct {
	Applied int    `json:"applied"`
	Failed  int    `json:"failed"`
	Message string `json:"message"`
}

// ApplyRDAChoices writes the chosen district onto each contact.
//
// Written as the AWARD OVERRIDE rather than over the imported CNTY, and the
// difference matters. CNTY is what the downloaded log said; it is a record of
// what HAMLOG holds, and overwriting it would destroy the very evidence the
// comparison was run on — the next comparison would then agree with itself and
// prove nothing. The override is the operator's decision, it is what the award
// engine counts, and it wins over both sources. So the log keeps saying what it
// said, the database keeps saying what it says, and the contact counts for the
// district the operator chose.
//
// A choice equal to what the database already answers is still written, on
// purpose: it records that the disagreement was looked at and settled, which a
// silent no-op would not.
func (a *App) ApplyRDAChoices(choices []RDAChoice) (RDAApplyResult, error) {
	var res RDAApplyResult
	if a.qso == nil {
		return res, fmt.Errorf("db not initialized")
	}
	for _, c := range choices {
		district := strings.ToUpper(strings.TrimSpace(c.District))
		if !rdaDistrictRe.MatchString(district) {
			res.Failed++
			applog.Printf("rda apply: qso %d — %q is not a district reference", c.QSOID, c.District)
			continue
		}
		q, err := a.qso.GetByID(a.ctx, c.QSOID)
		if err != nil {
			res.Failed++
			applog.Printf("rda apply: qso %d could not be read: %v", c.QSOID, err)
			continue
		}
		if q.Extras == nil {
			q.Extras = map[string]string{}
		}
		q.Extras[award.ManualRefsKey] = setOverrideRef(q.Extras[award.ManualRefsKey], "RDA", district)
		if err := a.qso.Update(a.ctx, q); err != nil {
			res.Failed++
			applog.Printf("rda apply: qso %d update failed: %v", c.QSOID, err)
			continue
		}
		res.Applied++
	}
	if res.Applied > 0 {
		// The award totals were computed from the old answers.
		a.invalidateAwardStats()
	}
	applog.Printf("rda apply: %d contacts settled, %d failed", res.Applied, res.Failed)
	res.Message = fmt.Sprintf("%d settled", res.Applied)
	if res.Failed > 0 {
		res.Message += fmt.Sprintf(", %d failed (see the log)", res.Failed)
	}
	return res, nil
}