feat(clublog): the matches come home, and Club Log lends its call list

Two new QSO columns, clublog_qso_download_status/date — a Club Log MATCH, the service's own confirmation (both stations uploaded the QSO, paired within 15 minutes). Full promoted-column lockstep: migration 0031, repo insert/scan, ADIF dictionary + import + export (app-defined CLUBLOG_QSO_DOWNLOAD_*), table columns, filter builder, bulk edit and the QSO editor's Club Log row. The QSL Manager's Club Log entry now actually downloads: getmatches.php with the existing account settings and the embedded application key, incremental via the match-completion date filter, matched call+band+mode ±15 min with a mode-blind fallback because Club Log reports 'false' for modes it cannot infer. Matches always exist on both sides, so unmatched ones are listed rather than skeleton-added. And Super Check Partial can merge Club Log's weekly SCP list (~180k calls worked on the air in the last 3 years) with MASTER.SCP — an opt-in checkbox under the SCP setting.
2026-08-31 10:14:52 +02:00
parent 1f667e4a4b
commit 3cb8096141
19 changed files with 464 additions and 64 deletions
@@ -13,6 +13,7 @@ package scp
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"fmt"
 	"io"
@@ -22,6 +23,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )

@@ -29,7 +31,14 @@ import (
 // line, '#'-prefixed header lines. ~50k+ active contest/DX calls.
 const masterURL = "https://www.supercheckpartial.com/MASTER.SCP"

+// clublogURL is Club Log's own SCP list, rebuilt weekly from real DX logs:
+// every call from a current entity with 40+ QSOs in the last 3 years (~180k).
+// Broader than MASTER.SCP (which leans contest), so the manager merges the two
+// when the Club Log source is enabled.
+const clublogURL = "https://cdn.clublog.org/clublog.scp.gz"
+
 const cacheFile = "MASTER.SCP"
+const clublogCacheFile = "CLUBLOG.SCP" // stored decompressed

 // Result is the two suggestion lists for a typed fragment.
 type Result struct {
@@ -44,6 +53,7 @@ type Manager struct {
 	updated time.Time // when the cache was last refreshed
 	dir     string
 	client  *http.Client
+	clublog atomic.Bool // merge Club Log's list into the master list
 }

 // NewManager loads any on-disk cache and returns a ready manager.
@@ -56,14 +66,30 @@ func NewManager(dataDir string) *Manager {
 	return m
 }

-func (m *Manager) path() string { return filepath.Join(m.dir, cacheFile) }
+func (m *Manager) path() string        { return filepath.Join(m.dir, cacheFile) }
+func (m *Manager) clublogPath() string { return filepath.Join(m.dir, clublogCacheFile) }
+
+// SetClublogEnabled turns the Club Log source on/off and reparses the on-disk
+// caches so the in-memory list reflects the choice immediately.
+func (m *Manager) SetClublogEnabled(on bool) {
+	if m.clublog.Swap(on) != on {
+		m.loadCache()
+	}
+}
+
+// ClublogEnabled reports whether the Club Log source is merged in.
+func (m *Manager) ClublogEnabled() bool { return m.clublog.Load() }

 func (m *Manager) loadCache() {
-	data, err := os.ReadFile(m.path())
-	if err != nil {
+	data, _ := os.ReadFile(m.path())
+	var extra []byte
+	if m.clublog.Load() {
+		extra, _ = os.ReadFile(m.clublogPath())
+	}
+	if data == nil && extra == nil {
 		return
 	}
-	m.parse(data)
+	m.parse(data, extra)
 	if fi, e := os.Stat(m.path()); e == nil {
 		m.mu.Lock()
 		m.updated = fi.ModTime()
@@ -91,7 +117,17 @@ func (m *Manager) Download(ctx context.Context) (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("scp: read: %w", err)
 	}
-	n := m.parse(body)
+	var extra []byte
+	if m.clublog.Load() {
+		if cl, cerr := m.downloadClublog(ctx); cerr == nil {
+			extra = cl
+		} else {
+			// The master list alone is still worth having; fall back to any
+			// cached Club Log list rather than dropping the source silently.
+			extra, _ = os.ReadFile(m.clublogPath())
+		}
+	}
+	n := m.parse(body, extra)
 	if n == 0 {
 		return 0, fmt.Errorf("scp: file parsed to 0 callsigns")
 	}
@@ -102,25 +138,57 @@ func (m *Manager) Download(ctx context.Context) (int, error) {
 	return n, nil
 }

+// downloadClublog fetches Club Log's gzipped SCP list and caches it decompressed.
+func (m *Manager) downloadClublog(ctx context.Context) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, clublogURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "OpsLog")
+	resp, err := m.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("scp: clublog request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("scp: clublog http %d", resp.StatusCode)
+	}
+	gz, err := gzip.NewReader(io.LimitReader(resp.Body, 32*1024*1024))
+	if err != nil {
+		return nil, fmt.Errorf("scp: clublog gunzip: %w", err)
+	}
+	body, err := io.ReadAll(io.LimitReader(gz, 64*1024*1024))
+	if err != nil {
+		return nil, fmt.Errorf("scp: clublog read: %w", err)
+	}
+	_ = os.WriteFile(m.clublogPath(), body, 0o644)
+	return body, nil
+}
+
 // parse loads the SCP bytes into the sorted call slice and returns the count.
-func (m *Manager) parse(data []byte) int {
+func (m *Manager) parse(datasets ...[]byte) int {
 	seen := make(map[string]struct{}, 1<<17)
-	sc := bufio.NewScanner(bytes.NewReader(data))
-	sc.Buffer(make([]byte, 1024*1024), 1024*1024)
-	for sc.Scan() {
-		line := strings.ToUpper(strings.TrimSpace(sc.Text()))
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue // blank / header comment
-		}
-		// A call token only (the master file is one call per line, but guard
-		// against stray trailing fields).
-		if i := strings.IndexAny(line, " \t,;"); i >= 0 {
-			line = line[:i]
-		}
-		if !plausibleCall(line) {
+	for _, data := range datasets {
+		if len(data) == 0 {
 			continue
 		}
-		seen[line] = struct{}{}
+		sc := bufio.NewScanner(bytes.NewReader(data))
+		sc.Buffer(make([]byte, 1024*1024), 1024*1024)
+		for sc.Scan() {
+			line := strings.ToUpper(strings.TrimSpace(sc.Text()))
+			if line == "" || strings.HasPrefix(line, "#") {
+				continue // blank / header comment
+			}
+			// A call token only (the files are one call per line, but guard
+			// against stray trailing fields).
+			if i := strings.IndexAny(line, " \t,;"); i >= 0 {
+				line = line[:i]
+			}
+			if !plausibleCall(line) {
+				continue
+			}
+			seen[line] = struct{}{}
+		}
 	}
 	if len(seen) == 0 {
 		return 0