feat(clublog): the matches come home, and Club Log lends its call list

Two new QSO columns, clublog_qso_download_status/date — a Club Log MATCH, the service's own confirmation (both stations uploaded the QSO, paired within 15 minutes). Full promoted-column lockstep: migration 0031, repo insert/scan, ADIF dictionary + import + export (app-defined CLUBLOG_QSO_DOWNLOAD_*), table columns, filter builder, bulk edit and the QSO editor's Club Log row. The QSL Manager's Club Log entry now actually downloads: getmatches.php with the existing account settings and the embedded application key, incremental via the match-completion date filter, matched call+band+mode ±15 min with a mode-blind fallback because Club Log reports 'false' for modes it cannot infer. Matches always exist on both sides, so unmatched ones are listed rather than skeleton-added. And Super Check Partial can merge Club Log's weekly SCP list (~180k calls worked on the air in the last 3 years) with MASTER.SCP — an opt-in checkbox under the SCP setting.
2026-08-31 10:14:52 +02:00
parent 1f667e4a4b
commit 3cb8096141
19 changed files with 464 additions and 64 deletions
@@ -329,6 +329,7 @@ const (
 	keyClusterSpotMax     = "cluster.spot_max"          // how many spots the list holds at once

 	keyScpEnabled = "scp.enabled" // Super Check Partial / N+1 suggestions on
+	keyScpClublog = "scp.clublog" // merge Club Log's weekly SCP list (~180k calls)

 	// Web publishing: the whole config as one JSON blob. A single key rather than
 	// twenty: it is read and written as a unit by one settings panel, and the FTP
@@ -421,6 +422,8 @@ const (
 	keyExtLoTWLastDownload = "extsvc.lotw.last_download" // YYYY-MM-DD of last confirmation pull
 	keyExtQRZLastDownload  = "extsvc.qrz.last_download"  // YYYY-MM-DD of last QRZ confirmation pull
 	keyExtEQSLLastDownload = "extsvc.eqsl.last_download" // YYYY-MM-DD of last eQSL inbox pull
+
+	keyExtClublogLastDownload = "extsvc.clublog.last_download" // YYYY-MM-DD of last Club Log matches pull
 )

 // QSLDefaults is the per-user default for the QSL / eQSL / LoTW / upload
@@ -1395,6 +1398,9 @@ func (a *App) startup(ctx context.Context) {
 	// Super Check Partial / N+1: load the cached MASTER.SCP; when the feature is
 	// enabled, auto-(re)download it if missing or older than a week.
 	a.scp = scp.NewManager(dataDir)
+	if v, _ := a.settings.Get(a.ctx, keyScpClublog); v == "1" {
+		a.scp.SetClublogEnabled(true)
+	}
 	go func() {
 		if v, _ := a.settings.Get(a.ctx, keyScpEnabled); v != "1" {
 			return
@@ -3146,6 +3152,7 @@ type ScpStatus struct {
 	Enabled bool   `json:"enabled"`
 	Count   int    `json:"count"`
 	Updated string `json:"updated,omitempty"` // RFC3339, empty if never
+	Clublog bool   `json:"clublog"`           // Club Log source merged in
 }

 // GetScpStatus returns whether SCP is enabled and how many calls are loaded.
@@ -3157,6 +3164,7 @@ func (a *App) GetScpStatus() ScpStatus {
 	}
 	if a.scp != nil {
 		st.Count = a.scp.Count()
+		st.Clublog = a.scp.ClublogEnabled()
 		if u := a.scp.Updated(); !u.IsZero() {
 			st.Updated = u.UTC().Format(time.RFC3339)
 		}
@@ -3164,6 +3172,34 @@ func (a *App) GetScpStatus() ScpStatus {
 	return st
 }

+// SetScpClublogEnabled merges (or stops merging) Club Log's weekly SCP list
+// into the Super Check Partial call list, and refreshes the download so the
+// change is visible without waiting for the weekly cycle.
+func (a *App) SetScpClublogEnabled(on bool) error {
+	if a.settings == nil {
+		return fmt.Errorf("db not initialized")
+	}
+	if err := a.settings.Set(a.ctx, keyScpClublog, boolStr(on)); err != nil {
+		return err
+	}
+	if a.scp != nil {
+		a.scp.SetClublogEnabled(on)
+		if on {
+			go func() {
+				if n, err := a.scp.Download(context.Background()); err == nil {
+					applog.Printf("scp: downloaded %d callsigns (with Club Log list)", n)
+					if a.ctx != nil {
+						wruntime.EventsEmit(a.ctx, "scp:updated")
+					}
+				} else {
+					applog.Printf("scp: download failed: %v", err)
+				}
+			}()
+		}
+	}
+	return nil
+}
+
 // SetScpEnabled turns Super Check Partial on/off. Enabling triggers a background
 // download when the list is missing or stale.
 func (a *App) SetScpEnabled(on bool) error {
@@ -6889,6 +6925,7 @@ var bulkFieldColumns = map[string]string{
 	"qrz_sent":     "qrzcom_qso_upload_status",
 	"qrz_rcvd":     "qrzcom_qso_download_status",
 	"clublog_sent": "clublog_qso_upload_status",
+	"clublog_rcvd": "clublog_qso_download_status",
 	"hrdlog_sent":  "hrdlog_qso_upload_status",
 	// Old ids kept so anything holding one keeps working.
 	"qrz_upload":     "qrzcom_qso_upload_status",
@@ -6905,6 +6942,7 @@ var bulkFieldColumns = map[string]string{
 	"qrz_sent_date":     "qrzcom_qso_upload_date",
 	"qrz_rcvd_date":     "qrzcom_qso_download_date",
 	"clublog_sent_date": "clublog_qso_upload_date",
+	"clublog_rcvd_date": "clublog_qso_download_date",
 	"hrdlog_sent_date":  "hrdlog_qso_upload_date",
 	// My station / operator
 	"station_callsign": "station_callsign",
@@ -12935,6 +12973,90 @@ func (a *App) runDownloadConfirmations(ctx context.Context, svc extsvc.Service,
 			a.setSetting(a.profileScope()+keyExtEQSLLastDownload, time.Now().UTC().Format("2006-01-02"))
 		}

+	case extsvc.ServiceClublog:
+		sinceDate := resolveSince(keyExtClublogLastDownload)
+		if sinceDate != "" {
+			emit("Downloading Club Log matches (matched since " + sinceDate + ")…")
+		} else {
+			emit("Downloading ALL Club Log matches (full pull — the date filter is faster next time)…")
+		}
+		matches, err := extsvc.DownloadClublogMatches(ctx, nil, cfg.Clublog, sinceDate)
+		if err != nil {
+			emit("Download failed: " + err.Error())
+			done(matched, total)
+			return
+		}
+		mIdx, kerr := a.qso.BuildMatchIndex(ctx, a.uploadOwnerCall(extsvc.ServiceClublog))
+		if kerr != nil {
+			emit("Error reading local log: " + kerr.Error())
+			done(matched, total)
+			return
+		}
+		// Club Log pairs QSOs within ±15 minutes, same as LoTW.
+		const clublogMatchWindow = 15 * time.Minute
+		today := time.Now().UTC().Format("20060102")
+		// A Club Log match is Club Log's own kind of confirmation, so NEW is
+		// judged only against other Club Log matches.
+		sets, _ := a.qso.ConfirmedSlots(ctx, []string{"clublog_qso_download_status"})
+		var items []ConfirmationItem
+		var unmatched []string
+		for _, m := range matches {
+			if ctx.Err() != nil {
+				return
+			}
+			total++
+			id, found := int64(0), false
+			if m.Band != "" && m.Mode != "" {
+				id, found = mIdx.Match(m.Callsign, m.Band, m.Mode, m.When, clublogMatchWindow)
+			}
+			if !found && m.Band != "" {
+				// Club Log reports "false" for modes it can't infer — fall back to
+				// call+band+time, still unambiguous inside a 15-minute window.
+				id, found = mIdx.MatchBand(m.Callsign, m.Band, m.When, clublogMatchWindow)
+			}
+			if found {
+				if e := a.qso.MarkClublogConfirmed(ctx, id, today); e == nil {
+					matched++
+				}
+			} else {
+				unmatched = append(unmatched, fmt.Sprintf("%s · %s · %s · %s",
+					m.Callsign, m.When.Format("2006-01-02 15:04Z"), m.Band, m.Mode))
+			}
+			it := ConfirmationItem{
+				Callsign: m.Callsign,
+				QSODate:  m.When.Format(time.RFC3339),
+				Band:     m.Band, Mode: m.Mode,
+			}
+			if m.DXCC != 0 {
+				it.Country = dxcc.NameForDXCC(m.DXCC)
+				it.NewDXCC = !sets.DXCC[m.DXCC]
+				it.NewBand = !sets.Band[qso.BandKey(m.DXCC, m.Band)]
+				it.NewMode = !sets.Mode[qso.ModeClassKey(m.DXCC, m.Mode)]
+				it.NewSlot = !sets.Slot[qso.SlotClassKey(m.DXCC, m.Band, m.Mode)]
+				sets.DXCC[m.DXCC] = true
+				sets.Band[qso.BandKey(m.DXCC, m.Band)] = true
+				sets.Mode[qso.ModeClassKey(m.DXCC, m.Mode)] = true
+				sets.Slot[qso.SlotClassKey(m.DXCC, m.Band, m.Mode)] = true
+			}
+			items = append(items, it)
+		}
+		if a.ctx != nil {
+			wruntime.EventsEmit(a.ctx, "qslmgr:confirmations", items)
+		}
+		emit(fmt.Sprintf("Matched %d of %d Club Log match(es)", matched, total))
+		if addNotFound && len(unmatched) > 0 {
+			// A match means BOTH sides uploaded this QSO — one Club Log can't find
+			// locally was deleted or edited here, not missed; adding a skeleton
+			// row from call/band/time alone would just create a duplicate shadow.
+			emit("Club Log matches carry too little detail to add missing QSOs — unmatched ones are listed below.")
+		}
+		for _, u := range unmatched {
+			emit("  ⚠ no local QSO for: " + u)
+		}
+		if a.settings != nil {
+			a.setSetting(a.profileScope()+keyExtClublogLastDownload, time.Now().UTC().Format("2006-01-02"))
+		}
+
 	default:
 		emit(fmt.Sprintf("Confirmation download isn't available for %s yet.", svc))
 	}