feat(clublog): the matches come home, and Club Log lends its call list
Two new QSO columns, clublog_qso_download_status/date — a Club Log MATCH, the service's own confirmation (both stations uploaded the QSO, paired within 15 minutes). Full promoted-column lockstep: migration 0031, repo insert/scan, ADIF dictionary + import + export (app-defined CLUBLOG_QSO_DOWNLOAD_*), table columns, filter builder, bulk edit and the QSO editor's Club Log row. The QSL Manager's Club Log entry now actually downloads: getmatches.php with the existing account settings and the embedded application key, incremental via the match-completion date filter, matched call+band+mode ±15 min with a mode-blind fallback because Club Log reports 'false' for modes it cannot infer. Matches always exist on both sides, so unmatched ones are listed rather than skeleton-added. And Super Check Partial can merge Club Log's weekly SCP list (~180k calls worked on the air in the last 3 years) with MASTER.SCP — an opt-in checkbox under the SCP setting.
This commit is contained in:
@@ -267,6 +267,8 @@ func writeRecord(bw *bufio.Writer, q qso.QSO, includeApp bool, allow map[string]
|
||||
w("QRZCOM_QSO_UPLOAD_STATUS", q.QRZComUploadStatus)
|
||||
w("QRZCOM_QSO_DOWNLOAD_DATE", q.QRZComDownloadDate)
|
||||
w("QRZCOM_QSO_DOWNLOAD_STATUS", q.QRZComDownloadStatus)
|
||||
w("CLUBLOG_QSO_DOWNLOAD_DATE", q.ClublogDownloadDate)
|
||||
w("CLUBLOG_QSO_DOWNLOAD_STATUS", q.ClublogDownloadStatus)
|
||||
|
||||
// --- Contest ---
|
||||
w("CONTEST_ID", q.ContestID)
|
||||
|
||||
@@ -161,6 +161,9 @@ var Fields = []FieldDef{
|
||||
{Name: "QRZCOM_QSO_UPLOAD_STATUS", Kind: KindEnum, Category: "QSL", Promoted: true},
|
||||
{Name: "QRZCOM_QSO_DOWNLOAD_DATE", Kind: KindDate, Category: "QSL", Promoted: true},
|
||||
{Name: "QRZCOM_QSO_DOWNLOAD_STATUS", Kind: KindEnum, Category: "QSL", Promoted: true},
|
||||
// App-defined pair (no standard ADIF field): Club Log's log-match download.
|
||||
{Name: "CLUBLOG_QSO_DOWNLOAD_DATE", Kind: KindDate, Category: "QSL", Promoted: true},
|
||||
{Name: "CLUBLOG_QSO_DOWNLOAD_STATUS", Kind: KindEnum, Category: "QSL", Promoted: true},
|
||||
{Name: "HAMLOGEU_QSO_UPLOAD_DATE", Kind: KindDate, Category: "QSL"},
|
||||
{Name: "HAMLOGEU_QSO_UPLOAD_STATUS", Kind: KindEnum, Category: "QSL"},
|
||||
{Name: "HAMQTH_QSO_UPLOAD_DATE", Kind: KindDate, Category: "QSL"},
|
||||
|
||||
@@ -287,6 +287,7 @@ var adifPromoted = stringSet(
|
||||
"hrdlog_qso_upload_date", "hrdlog_qso_upload_status",
|
||||
"qrzcom_qso_upload_date", "qrzcom_qso_upload_status",
|
||||
"qrzcom_qso_download_date", "qrzcom_qso_download_status",
|
||||
"clublog_qso_download_date", "clublog_qso_download_status",
|
||||
// Contest
|
||||
"contest_id", "srx", "stx", "srx_string", "stx_string",
|
||||
"check", "precedence", "arrl_sect",
|
||||
@@ -493,6 +494,8 @@ func recordToQSO(rec Record) (qso.QSO, bool) {
|
||||
q.QRZComUploadStatus = rec["qrzcom_qso_upload_status"]
|
||||
q.QRZComDownloadDate = rec["qrzcom_qso_download_date"]
|
||||
q.QRZComDownloadStatus = rec["qrzcom_qso_download_status"]
|
||||
q.ClublogDownloadDate = rec["clublog_qso_download_date"]
|
||||
q.ClublogDownloadStatus = rec["clublog_qso_download_status"]
|
||||
|
||||
// Contest
|
||||
q.ContestID = rec["contest_id"]
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Club Log log-matching (getmatches.php): a QSO both sides uploaded to Club
|
||||
-- Log is a confirmation in its own right. Mirrors qrzcom_qso_download_*.
|
||||
ALTER TABLE qso ADD COLUMN clublog_qso_download_date TEXT;
|
||||
ALTER TABLE qso ADD COLUMN clublog_qso_download_status TEXT;
|
||||
@@ -0,0 +1,126 @@
|
||||
package extsvc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Club Log's log-matching API. A "match" is a QSO that BOTH stations uploaded
|
||||
// to Club Log, paired within ±15 minutes — Club Log's own equivalent of a LoTW
|
||||
// confirmation. getmatches.php returns a JSON array of 5-element arrays:
|
||||
//
|
||||
// [["G0LGJ/M","223","2005-07-16 08:00:00","20","CW"], …]
|
||||
// callsign dxcc qso datetime (UTC) band mode ("false" when unknown)
|
||||
//
|
||||
// The optional start date filters on when Club Log COMPLETED the match (not
|
||||
// the QSO date), which is exactly what an incremental "since last download"
|
||||
// pull wants.
|
||||
const clublogMatchesURL = "https://clublog.org/getmatches.php"
|
||||
|
||||
// ClublogMatch is one confirmed pairing from getmatches.php.
|
||||
type ClublogMatch struct {
|
||||
Callsign string
|
||||
DXCC int
|
||||
When time.Time
|
||||
Band string // ADIF band ("20m", "70cm"); "" if the id is unknown
|
||||
Mode string // "" when Club Log doesn't know it
|
||||
}
|
||||
|
||||
// clublogBandNames maps Club Log's numeric band ids to ADIF band names. The
|
||||
// ids are the wavelength number; the only trap is that ids past the metre
|
||||
// bands are centimetres (the docs' own example: 70 = 70CM).
|
||||
var clublogBandNames = map[string]string{
|
||||
"2200": "2200m", "630": "630m", "160": "160m", "80": "80m", "60": "60m",
|
||||
"40": "40m", "30": "30m", "20": "20m", "17": "17m", "15": "15m",
|
||||
"12": "12m", "10": "10m", "8": "8m", "6": "6m", "5": "5m", "4": "4m",
|
||||
"2": "2m", "70": "70cm", "23": "23cm", "13": "13cm", "9": "9cm", "3": "3cm",
|
||||
}
|
||||
|
||||
// DownloadClublogMatches pulls the account's log matches for cfg.Callsign,
|
||||
// optionally only those Club Log completed since sinceDate ("2006-01-02").
|
||||
func DownloadClublogMatches(ctx context.Context, client *http.Client, cfg ServiceConfig, sinceDate string) ([]ClublogMatch, error) {
|
||||
email := strings.TrimSpace(cfg.Email)
|
||||
call := strings.ToUpper(strings.TrimSpace(cfg.Callsign))
|
||||
switch {
|
||||
case email == "":
|
||||
return nil, fmt.Errorf("clublog: account email not set")
|
||||
case cfg.Password == "":
|
||||
return nil, fmt.Errorf("clublog: password not set")
|
||||
case call == "":
|
||||
return nil, fmt.Errorf("clublog: callsign not set")
|
||||
}
|
||||
v := url.Values{}
|
||||
v.Set("api", clublogAppAPIKey)
|
||||
v.Set("email", email)
|
||||
v.Set("password", cfg.Password)
|
||||
v.Set("callsign", call)
|
||||
if t, err := time.Parse("2006-01-02", strings.TrimSpace(sinceDate)); err == nil {
|
||||
v.Set("startyear", strconv.Itoa(t.Year()))
|
||||
v.Set("startmonth", strconv.Itoa(int(t.Month())))
|
||||
v.Set("startday", strconv.Itoa(t.Day()))
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, clublogMatchesURL+"?"+v.Encode(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 120 * time.Second}
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
|
||||
text := strings.TrimSpace(string(body))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if looksLikeHTML(text) || len(text) > 300 {
|
||||
return nil, fmt.Errorf("clublog: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
return nil, fmt.Errorf("clublog: HTTP %d: %s", resp.StatusCode, text)
|
||||
}
|
||||
if looksLikeHTML(text) {
|
||||
return nil, fmt.Errorf("clublog: got a web page instead of matches — check email/password/callsign")
|
||||
}
|
||||
var raw [][]any
|
||||
if err := json.Unmarshal([]byte(text), &raw); err != nil {
|
||||
return nil, fmt.Errorf("clublog: bad matches JSON: %w", err)
|
||||
}
|
||||
str := func(x any) string {
|
||||
switch t := x.(type) {
|
||||
case string:
|
||||
return t
|
||||
case float64:
|
||||
return strconv.FormatFloat(t, 'f', -1, 64)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
out := make([]ClublogMatch, 0, len(raw))
|
||||
for _, rec := range raw {
|
||||
if len(rec) < 5 {
|
||||
continue
|
||||
}
|
||||
m := ClublogMatch{Callsign: strings.ToUpper(strings.TrimSpace(str(rec[0])))}
|
||||
m.DXCC, _ = strconv.Atoi(str(rec[1]))
|
||||
if t, err := time.Parse("2006-01-02 15:04:05", str(rec[2])); err == nil {
|
||||
m.When = t.UTC()
|
||||
}
|
||||
m.Band = clublogBandNames[strings.TrimSpace(str(rec[3]))]
|
||||
if md := strings.TrimSpace(str(rec[4])); md != "" && !strings.EqualFold(md, "false") {
|
||||
m.Mode = strings.ToUpper(md)
|
||||
}
|
||||
if m.Callsign == "" || m.When.IsZero() {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
+71
-31
@@ -123,6 +123,10 @@ type QSO struct {
|
||||
QRZComUploadStatus string `json:"qrzcom_qso_upload_status,omitempty"`
|
||||
QRZComDownloadDate string `json:"qrzcom_qso_download_date,omitempty"`
|
||||
QRZComDownloadStatus string `json:"qrzcom_qso_download_status,omitempty"`
|
||||
// Club Log match download (getmatches.php) — app-defined, no standard ADIF
|
||||
// field exists; exported/imported as CLUBLOG_QSO_DOWNLOAD_*.
|
||||
ClublogDownloadDate string `json:"clublog_qso_download_date,omitempty"`
|
||||
ClublogDownloadStatus string `json:"clublog_qso_download_status,omitempty"`
|
||||
|
||||
// --- Contest ---
|
||||
ContestID string `json:"contest_id,omitempty"`
|
||||
@@ -261,6 +265,7 @@ const columnList = `callsign, qso_date, qso_date_off, band, band_rx, mode, submo
|
||||
hrdlog_qso_upload_date, hrdlog_qso_upload_status,
|
||||
qrzcom_qso_upload_date, qrzcom_qso_upload_status,
|
||||
qrzcom_qso_download_date, qrzcom_qso_download_status,
|
||||
clublog_qso_download_date, clublog_qso_download_status,
|
||||
contest_id, srx, stx, srx_string, stx_string, check_field, precedence, arrl_sect,
|
||||
prop_mode, sat_name, sat_mode, ant_az, ant_el, ant_path,
|
||||
station_callsign, operator, my_grid, my_gridsquare_ext, my_country, my_state, my_cnty, my_iota,
|
||||
@@ -336,6 +341,7 @@ func (q *QSO) args() []any {
|
||||
q.HRDLogUploadDate, q.HRDLogUploadStatus,
|
||||
q.QRZComUploadDate, q.QRZComUploadStatus,
|
||||
q.QRZComDownloadDate, q.QRZComDownloadStatus,
|
||||
q.ClublogDownloadDate, q.ClublogDownloadStatus,
|
||||
q.ContestID, q.SRX, q.STX, q.SRXString, q.STXString, q.Check, q.Precedence, q.ARRLSect,
|
||||
q.PropMode, q.SatName, q.SatMode, q.AntAz, q.AntEl, q.AntPath,
|
||||
q.StationCallsign, q.Operator, q.MyGrid, q.MyGridExt, q.MyCountry, q.MyState, q.MyCounty, q.MyIOTA,
|
||||
@@ -771,30 +777,32 @@ func (r *Repo) MarkEQSLSent(ctx context.Context, id int64, date string) error {
|
||||
// zones, lat/lon) that are meaningless shared.
|
||||
var bulkEditableCols = map[string]bool{
|
||||
// QSL / upload status
|
||||
"lotw_sent": true,
|
||||
"lotw_rcvd": true,
|
||||
"eqsl_sent": true,
|
||||
"eqsl_rcvd": true,
|
||||
"qsl_sent": true,
|
||||
"qsl_rcvd": true,
|
||||
"qsl_via": true,
|
||||
"qsl_sent_via": true,
|
||||
"qsl_rcvd_via": true,
|
||||
"qrzcom_qso_upload_status": true,
|
||||
"qrzcom_qso_download_status": true,
|
||||
"clublog_qso_upload_status": true,
|
||||
"hrdlog_qso_upload_status": true,
|
||||
"lotw_sent": true,
|
||||
"lotw_rcvd": true,
|
||||
"eqsl_sent": true,
|
||||
"eqsl_rcvd": true,
|
||||
"qsl_sent": true,
|
||||
"qsl_rcvd": true,
|
||||
"qsl_via": true,
|
||||
"qsl_sent_via": true,
|
||||
"qsl_rcvd_via": true,
|
||||
"qrzcom_qso_upload_status": true,
|
||||
"qrzcom_qso_download_status": true,
|
||||
"clublog_qso_upload_status": true,
|
||||
"clublog_qso_download_status": true,
|
||||
"hrdlog_qso_upload_status": true,
|
||||
// Confirmation DATES. ADIF YYYYMMDD strings, so plain TEXT like the rest.
|
||||
"qsl_sent_date": true,
|
||||
"qsl_rcvd_date": true,
|
||||
"lotw_sent_date": true,
|
||||
"lotw_rcvd_date": true,
|
||||
"eqsl_sent_date": true,
|
||||
"eqsl_rcvd_date": true,
|
||||
"qrzcom_qso_upload_date": true,
|
||||
"qrzcom_qso_download_date": true,
|
||||
"clublog_qso_upload_date": true,
|
||||
"hrdlog_qso_upload_date": true,
|
||||
"qsl_sent_date": true,
|
||||
"qsl_rcvd_date": true,
|
||||
"lotw_sent_date": true,
|
||||
"lotw_rcvd_date": true,
|
||||
"eqsl_sent_date": true,
|
||||
"eqsl_rcvd_date": true,
|
||||
"qrzcom_qso_upload_date": true,
|
||||
"qrzcom_qso_download_date": true,
|
||||
"clublog_qso_upload_date": true,
|
||||
"clublog_qso_download_date": true,
|
||||
"hrdlog_qso_upload_date": true,
|
||||
// My station / operator
|
||||
"station_callsign": true,
|
||||
"operator": true,
|
||||
@@ -1339,15 +1347,17 @@ var filterableColumns = map[string]bool{
|
||||
"qsl_sent": true, "qsl_rcvd": true, "qsl_via": true, "qsl_sent_via": true, "qsl_rcvd_via": true,
|
||||
"lotw_sent": true, "lotw_rcvd": true, "eqsl_sent": true, "eqsl_rcvd": true,
|
||||
"qrzcom_qso_upload_status": true, "qrzcom_qso_download_status": true,
|
||||
"clublog_qso_upload_status": true, "hrdlog_qso_upload_status": true,
|
||||
"clublog_qso_upload_status": true, "clublog_qso_download_status": true,
|
||||
"hrdlog_qso_upload_status": true,
|
||||
// Confirmation DATES. ADIF YYYYMMDD strings, so a plain string comparison is
|
||||
// also chronological — "before 20240101" works with no date parsing.
|
||||
"qsl_sent_date": true, "qsl_rcvd_date": true,
|
||||
"lotw_sent_date": true, "lotw_rcvd_date": true,
|
||||
"eqsl_sent_date": true, "eqsl_rcvd_date": true,
|
||||
"qrzcom_qso_upload_date": true, "qrzcom_qso_download_date": true,
|
||||
"clublog_qso_upload_date": true, "hrdlog_qso_upload_date": true,
|
||||
"contest_id": true, "srx": true, "stx": true,
|
||||
"clublog_qso_upload_date": true, "clublog_qso_download_date": true,
|
||||
"hrdlog_qso_upload_date": true,
|
||||
"contest_id": true, "srx": true, "stx": true,
|
||||
"prop_mode": true, "sat_name": true,
|
||||
"station_callsign": true, "operator": true, "my_grid": true, "my_country": true,
|
||||
"my_state": true, "my_cnty": true, "my_iota": true, "my_sota_ref": true, "my_pota_ref": true,
|
||||
@@ -3264,6 +3274,7 @@ type matchRef struct {
|
||||
// FT8, FT4 only FT4, CW only CW…). Built in one table scan.
|
||||
type MatchIndex struct {
|
||||
byMode map[string][]matchRef // call|band|canonMode → refs
|
||||
byBand map[string][]matchRef // call|band → refs (mode-blind fallback)
|
||||
}
|
||||
|
||||
// canonMode folds the phone sidebands into a single "SSB" bucket; every other
|
||||
@@ -3299,7 +3310,7 @@ func parseQSODate(s string) time.Time {
|
||||
// for one of the operator's calls (e.g. F4BPO) never touches QSOs logged under
|
||||
// another (e.g. TM2Q).
|
||||
func (r *Repo) BuildMatchIndex(ctx context.Context, ownerCall string) (*MatchIndex, error) {
|
||||
idx := &MatchIndex{byMode: map[string][]matchRef{}}
|
||||
idx := &MatchIndex{byMode: map[string][]matchRef{}, byBand: map[string][]matchRef{}}
|
||||
query := `SELECT id, callsign, qso_date, band, mode FROM qso`
|
||||
var args []any
|
||||
if oc := strings.ToUpper(strings.TrimSpace(ownerCall)); oc != "" {
|
||||
@@ -3327,6 +3338,11 @@ func (r *Repo) BuildMatchIndex(ctx context.Context, ownerCall string) (*MatchInd
|
||||
func (idx *MatchIndex) add(call, band, mode string, when time.Time, id int64) {
|
||||
mk := matchKeyMode(call, band, mode)
|
||||
idx.byMode[mk] = append(idx.byMode[mk], matchRef{when: when, id: id})
|
||||
if idx.byBand == nil { // tests build the index literally, without byBand
|
||||
idx.byBand = map[string][]matchRef{}
|
||||
}
|
||||
bk := strings.ToUpper(call) + "|" + strings.ToLower(band)
|
||||
idx.byBand[bk] = append(idx.byBand[bk], matchRef{when: when, id: id})
|
||||
}
|
||||
|
||||
// Add registers a QSO in the index (exported wrapper for callers that inserted a
|
||||
@@ -3342,6 +3358,12 @@ func (idx *MatchIndex) Match(call, band, mode string, when time.Time, window tim
|
||||
return closestRef(idx.byMode[matchKeyMode(call, band, mode)], when, window)
|
||||
}
|
||||
|
||||
// MatchBand matches ignoring the mode — for confirmations whose source doesn't
|
||||
// carry one (Club Log reports "false" for modes it can't infer).
|
||||
func (idx *MatchIndex) MatchBand(call, band string, when time.Time, window time.Duration) (int64, bool) {
|
||||
return closestRef(idx.byBand[strings.ToUpper(call)+"|"+strings.ToLower(band)], when, window)
|
||||
}
|
||||
|
||||
func closestRef(refs []matchRef, when time.Time, window time.Duration) (int64, bool) {
|
||||
var best int64
|
||||
bestD := window + time.Second
|
||||
@@ -3500,10 +3522,11 @@ func (r *Repo) GetSlotStats(ctx context.Context) (SlotStats, error) {
|
||||
// confirmedCols whitelists the received-status columns ConfirmedSlots may
|
||||
// OR together (guards the dynamic SQL).
|
||||
var confirmedCols = map[string]bool{
|
||||
"lotw_rcvd": true,
|
||||
"qsl_rcvd": true,
|
||||
"eqsl_rcvd": true,
|
||||
"qrzcom_qso_download_status": true,
|
||||
"lotw_rcvd": true,
|
||||
"qsl_rcvd": true,
|
||||
"eqsl_rcvd": true,
|
||||
"qrzcom_qso_download_status": true,
|
||||
"clublog_qso_download_status": true,
|
||||
}
|
||||
|
||||
// ConfirmedSlots returns the set of confirmed DXCC/band/slot combos, counting
|
||||
@@ -3560,6 +3583,19 @@ func (r *Repo) MarkQRZConfirmed(ctx context.Context, id int64, date string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkClublogConfirmed stamps CLUBLOG_QSO_DOWNLOAD_STATUS=Y and the date on a
|
||||
// QSO Club Log reports as matched. date is an ADIF YYYYMMDD string.
|
||||
func (r *Repo) MarkClublogConfirmed(ctx context.Context, id int64, date string) error {
|
||||
_, err := r.db.ExecContext(ctx,
|
||||
`UPDATE qso SET clublog_qso_download_status = 'Y', clublog_qso_download_date = ?,
|
||||
updated_at = ? WHERE id = ?`,
|
||||
date, db.NowISO(), id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mark clublog confirmed %d: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearQRZConfirmed takes back a QRZ confirmation.
|
||||
//
|
||||
// Needed because OpsLog set some wrongly: it read qrzcom_qso_download_status,
|
||||
@@ -3638,6 +3674,7 @@ func scanQSO(s scanner) (QSO, error) {
|
||||
hrdlogDate, hrdlogStatus sql.NullString
|
||||
qrzcomDate, qrzcomStatus sql.NullString
|
||||
qrzcomDlDate, qrzcomDlStatus sql.NullString
|
||||
clublogDlDate, clublogDlStatus sql.NullString
|
||||
contestID sql.NullString
|
||||
srx, stx sql.NullInt64
|
||||
srxStr, stxStr sql.NullString
|
||||
@@ -3682,6 +3719,7 @@ func scanQSO(s scanner) (QSO, error) {
|
||||
&hrdlogDate, &hrdlogStatus,
|
||||
&qrzcomDate, &qrzcomStatus,
|
||||
&qrzcomDlDate, &qrzcomDlStatus,
|
||||
&clublogDlDate, &clublogDlStatus,
|
||||
&contestID, &srx, &stx, &srxStr, &stxStr, &checkField, &precedence, &arrlSect,
|
||||
&propMode, &satName, &satMode, &antAz, &antEl, &antPath,
|
||||
&stCall, &op, &myGrid, &myGridExt, &myCountry, &myState, &myCnty, &myIOTA,
|
||||
@@ -3779,6 +3817,8 @@ func scanQSO(s scanner) (QSO, error) {
|
||||
q.QRZComUploadStatus = qrzcomStatus.String
|
||||
q.QRZComDownloadDate = qrzcomDlDate.String
|
||||
q.QRZComDownloadStatus = qrzcomDlStatus.String
|
||||
q.ClublogDownloadDate = clublogDlDate.String
|
||||
q.ClublogDownloadStatus = clublogDlStatus.String
|
||||
q.ContestID = contestID.String
|
||||
if srx.Valid {
|
||||
v := int(srx.Int64)
|
||||
|
||||
+88
-20
@@ -13,6 +13,7 @@ package scp
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -29,7 +31,14 @@ import (
|
||||
// line, '#'-prefixed header lines. ~50k+ active contest/DX calls.
|
||||
const masterURL = "https://www.supercheckpartial.com/MASTER.SCP"
|
||||
|
||||
// clublogURL is Club Log's own SCP list, rebuilt weekly from real DX logs:
|
||||
// every call from a current entity with 40+ QSOs in the last 3 years (~180k).
|
||||
// Broader than MASTER.SCP (which leans contest), so the manager merges the two
|
||||
// when the Club Log source is enabled.
|
||||
const clublogURL = "https://cdn.clublog.org/clublog.scp.gz"
|
||||
|
||||
const cacheFile = "MASTER.SCP"
|
||||
const clublogCacheFile = "CLUBLOG.SCP" // stored decompressed
|
||||
|
||||
// Result is the two suggestion lists for a typed fragment.
|
||||
type Result struct {
|
||||
@@ -44,6 +53,7 @@ type Manager struct {
|
||||
updated time.Time // when the cache was last refreshed
|
||||
dir string
|
||||
client *http.Client
|
||||
clublog atomic.Bool // merge Club Log's list into the master list
|
||||
}
|
||||
|
||||
// NewManager loads any on-disk cache and returns a ready manager.
|
||||
@@ -56,14 +66,30 @@ func NewManager(dataDir string) *Manager {
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Manager) path() string { return filepath.Join(m.dir, cacheFile) }
|
||||
func (m *Manager) path() string { return filepath.Join(m.dir, cacheFile) }
|
||||
func (m *Manager) clublogPath() string { return filepath.Join(m.dir, clublogCacheFile) }
|
||||
|
||||
// SetClublogEnabled turns the Club Log source on/off and reparses the on-disk
|
||||
// caches so the in-memory list reflects the choice immediately.
|
||||
func (m *Manager) SetClublogEnabled(on bool) {
|
||||
if m.clublog.Swap(on) != on {
|
||||
m.loadCache()
|
||||
}
|
||||
}
|
||||
|
||||
// ClublogEnabled reports whether the Club Log source is merged in.
|
||||
func (m *Manager) ClublogEnabled() bool { return m.clublog.Load() }
|
||||
|
||||
func (m *Manager) loadCache() {
|
||||
data, err := os.ReadFile(m.path())
|
||||
if err != nil {
|
||||
data, _ := os.ReadFile(m.path())
|
||||
var extra []byte
|
||||
if m.clublog.Load() {
|
||||
extra, _ = os.ReadFile(m.clublogPath())
|
||||
}
|
||||
if data == nil && extra == nil {
|
||||
return
|
||||
}
|
||||
m.parse(data)
|
||||
m.parse(data, extra)
|
||||
if fi, e := os.Stat(m.path()); e == nil {
|
||||
m.mu.Lock()
|
||||
m.updated = fi.ModTime()
|
||||
@@ -91,7 +117,17 @@ func (m *Manager) Download(ctx context.Context) (int, error) {
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("scp: read: %w", err)
|
||||
}
|
||||
n := m.parse(body)
|
||||
var extra []byte
|
||||
if m.clublog.Load() {
|
||||
if cl, cerr := m.downloadClublog(ctx); cerr == nil {
|
||||
extra = cl
|
||||
} else {
|
||||
// The master list alone is still worth having; fall back to any
|
||||
// cached Club Log list rather than dropping the source silently.
|
||||
extra, _ = os.ReadFile(m.clublogPath())
|
||||
}
|
||||
}
|
||||
n := m.parse(body, extra)
|
||||
if n == 0 {
|
||||
return 0, fmt.Errorf("scp: file parsed to 0 callsigns")
|
||||
}
|
||||
@@ -102,25 +138,57 @@ func (m *Manager) Download(ctx context.Context) (int, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// downloadClublog fetches Club Log's gzipped SCP list and caches it decompressed.
|
||||
func (m *Manager) downloadClublog(ctx context.Context) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, clublogURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "OpsLog")
|
||||
resp, err := m.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scp: clublog request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("scp: clublog http %d", resp.StatusCode)
|
||||
}
|
||||
gz, err := gzip.NewReader(io.LimitReader(resp.Body, 32*1024*1024))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scp: clublog gunzip: %w", err)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(gz, 64*1024*1024))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scp: clublog read: %w", err)
|
||||
}
|
||||
_ = os.WriteFile(m.clublogPath(), body, 0o644)
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// parse loads the SCP bytes into the sorted call slice and returns the count.
|
||||
func (m *Manager) parse(data []byte) int {
|
||||
func (m *Manager) parse(datasets ...[]byte) int {
|
||||
seen := make(map[string]struct{}, 1<<17)
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
sc.Buffer(make([]byte, 1024*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.ToUpper(strings.TrimSpace(sc.Text()))
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // blank / header comment
|
||||
}
|
||||
// A call token only (the master file is one call per line, but guard
|
||||
// against stray trailing fields).
|
||||
if i := strings.IndexAny(line, " \t,;"); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
if !plausibleCall(line) {
|
||||
for _, data := range datasets {
|
||||
if len(data) == 0 {
|
||||
continue
|
||||
}
|
||||
seen[line] = struct{}{}
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
sc.Buffer(make([]byte, 1024*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.ToUpper(strings.TrimSpace(sc.Text()))
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue // blank / header comment
|
||||
}
|
||||
// A call token only (the files are one call per line, but guard
|
||||
// against stray trailing fields).
|
||||
if i := strings.IndexAny(line, " \t,;"); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
if !plausibleCall(line) {
|
||||
continue
|
||||
}
|
||||
seen[line] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return 0
|
||||
|
||||
@@ -190,6 +190,8 @@ var Columns = []Column{
|
||||
{"eqsl_rcvd_date", "Eqsl rcvd date", "QSL", func(q *qso.QSO) string { return q.EQSLRcvdDate }},
|
||||
{"clublog_qso_upload_date", "Clublog qso upload date", "QSL", func(q *qso.QSO) string { return q.ClublogUploadDate }},
|
||||
{"clublog_qso_upload_status", "Clublog qso upload status", "QSL", func(q *qso.QSO) string { return q.ClublogUploadStatus }},
|
||||
{"clublog_qso_download_date", "Clublog match date", "QSL", func(q *qso.QSO) string { return q.ClublogDownloadDate }},
|
||||
{"clublog_qso_download_status", "Clublog match status", "QSL", func(q *qso.QSO) string { return q.ClublogDownloadStatus }},
|
||||
{"hrdlog_qso_upload_date", "Hrdlog qso upload date", "QSL", func(q *qso.QSO) string { return q.HRDLogUploadDate }},
|
||||
{"hrdlog_qso_upload_status", "Hrdlog qso upload status", "QSL", func(q *qso.QSO) string { return q.HRDLogUploadStatus }},
|
||||
{"qrzcom_qso_upload_date", "Qrzcom qso upload date", "QSL", func(q *qso.QSO) string { return q.QRZComUploadDate }},
|
||||
|
||||
Reference in New Issue
Block a user