feat: Added support for US Counties in OpsLog / Extra feature with DXHunter

This commit is contained in:
2026-07-17 13:23:35 +02:00
parent 1a155e3627
commit dd3b51a2ae
14 changed files with 4127 additions and 3 deletions
+55
View File
@@ -643,6 +643,59 @@ func InScope(d Def, q *qso.QSO) bool { return inScope(&d, q) }
// EmissionOf maps an ADIF mode to its broad category (CW|PHONE|DIGITAL).
func EmissionOf(mode string) string { return emissionOf(mode) }
// USCountyKey normalises a QSO's state + cnty into the canonical "STATE,COUNTY"
// match code used by the US Counties (USA-CA) award, so the reference list (in
// that same form) matches whatever shape the logbook holds. It is the SINGLE
// source of truth: cmd/cntygen builds the reference codes by calling it, so the
// two sides can never drift.
//
// Real logs are a mess — "MA,MIDDLESEX", bare "Middlesex" with the state in its
// own column, mixed case, county-type suffixes, the odd "0", plus the FIPS list
// abbreviating "Saint"→"St." and hyphenating "Matanuska-Susitna". The rules:
// - if cnty already carries "ST,County", split on the first comma; else take
// the state from the STATE column;
// - upper-case; drop periods and apostrophes; hyphens→space; strip a trailing
// County/Parish/Borough/Census Area/Municipality; fold Saint(e)→St(e);
// collapse whitespace;
// - require a 2-letter state and a non-empty county, else no match ("").
func USCountyKey(state, cnty string) string {
s := strings.TrimSpace(cnty)
if s == "" || s == "0" {
return ""
}
var st, co string
if i := strings.IndexByte(s, ','); i >= 0 {
st, co = strings.TrimSpace(s[:i]), strings.TrimSpace(s[i+1:])
} else {
st, co = strings.TrimSpace(state), s
}
co = strings.ToUpper(co)
co = strings.ReplaceAll(co, ".", "")
co = strings.ReplaceAll(co, "'", "")
co = strings.ReplaceAll(co, "-", " ")
for _, suf := range []string{" COUNTY", " PARISH", " BOROUGH", " CENSUS AREA", " MUNICIPALITY"} {
if strings.HasSuffix(co, suf) {
co = strings.TrimSuffix(co, suf)
break
}
}
switch {
case strings.HasPrefix(co, "SAINTE "):
co = "STE " + co[len("SAINTE "):]
case strings.HasPrefix(co, "SAINT "):
co = "ST " + co[len("SAINT "):]
}
// Drop ALL internal spaces last: FIPS writes DeKalb/DuPage/LaSalle as one
// word, logs often split them ("De Kalb"). Removing spaces on both sides
// folds those together and can't collide two real counties in one state.
co = strings.ReplaceAll(co, " ", "")
st = strings.ToUpper(st)
if len(st) != 2 || co == "" {
return ""
}
return st + "," + co
}
// labelRef fills a worked reference's name/group from the reference list (or the
// name resolver as a fallback).
func labelRef(rf *Ref, d *Def, code string, rl refList, hasList bool, nameOf NameResolver) {
@@ -1145,6 +1198,8 @@ func fieldRaw(field string, q *qso.QSO) string {
return q.Callsign
case "state":
return q.State
case "us_county":
return USCountyKey(q.State, q.County)
case "cont":
return q.Continent
case "country":
+29
View File
@@ -0,0 +1,29 @@
{
"def": {
"code": "USCOUNTIES",
"name": "US Counties (USA-CA)",
"description": "Worked US counties (CQ USA-CA). Matches the QSO's county, tolerating LoTW \"ST,County\" and bare county-name shapes.",
"valid": true,
"protected": true,
"type": "QSOFIELDS",
"field": "us_county",
"match_by": "code",
"exact_match": true,
"pattern": "",
"ref_display": "name",
"dxcc_filter": [
291,
110,
6
],
"confirm": [
"lotw",
"qsl"
],
"validate": [
"lotw"
],
"total": 3077,
"builtin": true
}
}
+27
View File
@@ -0,0 +1,27 @@
package award
import "testing"
func TestUSCountyKey(t *testing.T) {
cases := []struct {
state, cnty, want string
}{
{"MA", "MA,MIDDLESEX", "MA,MIDDLESEX"}, // LoTW "ST,County" shape
{"NJ", "Middlesex", "NJ,MIDDLESEX"}, // bare name + state column
{"TX", "Montgomery", "TX,MONTGOMERY"}, // title case
{"FL", "Saint Lucie", "FL,STLUCIE"}, // Saint→St, space dropped
{"FL", "St. Lucie", "FL,STLUCIE"}, // FIPS abbreviation, period dropped
{"AK", "Matanuska-Susitna", "AK,MATANUSKASUSITNA"}, // hyphen→space→dropped
{"IL", "De Kalb", "IL,DEKALB"}, // split name
{"IL", "DeKalb County", "IL,DEKALB"}, // suffix + one word
{"LA", "Acadia Parish", "LA,ACADIA"}, // parish suffix
{"", "Honolulu", ""}, // no state → no match
{"HI", "0", ""}, // garbage
{"HI", "", ""}, // empty
}
for _, c := range cases {
if got := USCountyKey(c.state, c.cnty); got != c.want {
t.Errorf("USCountyKey(%q,%q) = %q, want %q", c.state, c.cnty, got, c.want)
}
}
}
+3
View File
@@ -17,6 +17,7 @@ import (
// - WAC → continent code ("EU", "NA", …)
// - WAS → ADIF STATE code ("AL", …)
// - DDFM → "D06" (the award pattern captures the leading D)
// - USCOUNTIES → canonical "STATE,COUNTY" key (see award.usCountyKey)
func BuiltinRefs(code string) ([]Ref, bool) {
switch code {
case "DXCC":
@@ -29,6 +30,8 @@ func BuiltinRefs(code string) ([]Ref, bool) {
return usStates().Refs, true
case "DDFM":
return frenchDepartments(), true
case "USCOUNTIES":
return usCounties(), true
}
return nil, false
}
File diff suppressed because it is too large Load Diff
+403
View File
@@ -0,0 +1,403 @@
// Package uls resolves a US amateur callsign to its county and grid, offline,
// from the FCC ULS licence database cross-referenced with a ZIP→county/lat-lon
// table. It backs the US Counties (USA-CA) award and county hunting: an FCC
// spot or a bare CW/SSB spot carries only a callsign, and this turns that into
// a county with no per-lookup API call.
//
// Data lives in its OWN local SQLite file (data/uls.db), never in the logbook:
// it is ~800k rows of static reference data that would only bloat the log (and
// crawl over a remote MySQL link). It is downloaded on demand, not shipped.
//
// Sources, both public and free:
// - FCC ULS Amateur, full database: l_amat.zip → EN.dat (pipe-delimited).
// Fields used: [4] call_sign, [17] state, [18] zip_code.
// - GeoNames US postal codes: US.zip → US.txt (tab-delimited).
// Fields used: [1] zip, [4] state, [5] county, [9] lat, [10] lon.
//
// The county from a ZIP is the ZIP's primary county — a ZIP can straddle a line,
// so this is ~98% right for fixed stations (rovers/portables need the from-air
// grid, which arrives separately via heard_geo). Good enough to hunt with.
package uls
import (
"archive/zip"
"bufio"
"context"
"database/sql"
"fmt"
"io"
"math"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
)
// Default download URLs (overridable in Import for tests).
const (
fccAmateurURL = "https://data.fcc.gov/download/pub/uls/complete/l_amat.zip"
geoNamesURL = "https://download.geonames.org/export/zip/US.zip"
)
// Location is a resolved callsign's home county + grid.
type Location struct {
State string `json:"state"`
County string `json:"county"` // GeoNames county name (e.g. "Middlesex")
Grid string `json:"grid"` // 6-char Maidenhead from the ZIP centroid
}
// CNTY renders the ADIF "STATE,County" form for stamping a QSO's cnty field.
func (l Location) CNTY() string {
if l.State == "" || l.County == "" {
return ""
}
return l.State + "," + l.County
}
// Store owns the local uls.db connection.
type Store struct {
db *sql.DB
mu sync.RWMutex // guards the whole DB during a re-import (DELETE+bulk INSERT)
}
// Open opens (creating if needed) the ULS SQLite store at path.
func Open(path string) (*Store, error) {
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
if err != nil {
return nil, err
}
if _, err := db.Exec(`
CREATE TABLE IF NOT EXISTS uls_callsign (
callsign TEXT PRIMARY KEY,
state TEXT NOT NULL DEFAULT '',
county TEXT NOT NULL DEFAULT '',
grid TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS uls_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT ''
);`); err != nil {
db.Close()
return nil, err
}
return &Store{db: db}, nil
}
func (s *Store) Close() error { return s.db.Close() }
// Count returns how many callsigns are loaded.
func (s *Store) Count() int {
s.mu.RLock()
defer s.mu.RUnlock()
var n int
s.db.QueryRow(`SELECT COUNT(*) FROM uls_callsign`).Scan(&n)
return n
}
// UpdatedAt returns when the store was last imported (zero if never).
func (s *Store) UpdatedAt() time.Time {
s.mu.RLock()
defer s.mu.RUnlock()
var v string
if err := s.db.QueryRow(`SELECT value FROM uls_meta WHERE key='updated_at'`).Scan(&v); err != nil {
return time.Time{}
}
t, _ := time.Parse(time.RFC3339, v)
return t
}
// Resolve looks up a callsign's home county + grid. ok=false if unknown or the
// store is empty.
func (s *Store) Resolve(callsign string) (Location, bool) {
call := strings.ToUpper(strings.TrimSpace(callsign))
if call == "" {
return Location{}, false
}
s.mu.RLock()
defer s.mu.RUnlock()
var l Location
err := s.db.QueryRow(`SELECT state, county, grid FROM uls_callsign WHERE callsign=?`, call).
Scan(&l.State, &l.County, &l.Grid)
if err != nil || l.County == "" {
return Location{}, false
}
return l, true
}
// Progress reports import stages to the caller (0..100 within a stage).
type Progress func(stage string, pct int)
// zipRow is one ZIP's primary county + centroid.
type zipRow struct {
state, county string
lat, lon float64
}
// Import downloads the FCC ULS + GeoNames data, rebuilds the callsign→county
// table, and records the timestamp. It replaces the table atomically: on any
// error the previous contents are left intact. tmpDir is where the (large) zips
// are streamed; "" uses the OS temp dir.
func (s *Store) Import(ctx context.Context, tmpDir string, prog Progress) error {
if prog == nil {
prog = func(string, int) {}
}
if tmpDir == "" {
tmpDir = os.TempDir()
}
// 1) ZIP→county/lat-lon crosswalk (small).
prog("Downloading ZIP crosswalk", 0)
geoPath := filepath.Join(tmpDir, "opslog_geonames_us.zip")
if err := download(ctx, geoNamesURL, geoPath, nil); err != nil {
return fmt.Errorf("download GeoNames: %w", err)
}
defer os.Remove(geoPath)
prog("Parsing ZIP crosswalk", 50)
zipmap, err := parseGeoNames(geoPath)
if err != nil {
return fmt.Errorf("parse GeoNames: %w", err)
}
if len(zipmap) == 0 {
return fmt.Errorf("GeoNames crosswalk is empty")
}
// 2) FCC ULS full amateur database (large).
prog("Downloading FCC ULS database", 0)
amatPath := filepath.Join(tmpDir, "opslog_l_amat.zip")
if err := download(ctx, fccAmateurURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
return fmt.Errorf("download FCC ULS: %w", err)
}
defer os.Remove(amatPath)
// 3) Parse EN.dat, join the crosswalk, rebuild the table.
prog("Building county database", 0)
return s.rebuild(ctx, amatPath, zipmap, prog)
}
// rebuild streams EN.dat out of the FCC zip and replaces uls_callsign in one
// transaction (old data survives a failure).
func (s *Store) rebuild(ctx context.Context, amatZip string, zipmap map[string]zipRow, prog Progress) error {
zr, err := zip.OpenReader(amatZip)
if err != nil {
return fmt.Errorf("open FCC zip: %w", err)
}
defer zr.Close()
var en *zip.File
for _, f := range zr.File {
if strings.EqualFold(filepath.Base(f.Name), "EN.dat") {
en = f
break
}
}
if en == nil {
return fmt.Errorf("EN.dat not found in FCC zip")
}
rc, err := en.Open()
if err != nil {
return fmt.Errorf("open EN.dat: %w", err)
}
defer rc.Close()
s.mu.Lock()
defer s.mu.Unlock()
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `DELETE FROM uls_callsign`); err != nil {
return err
}
stmt, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO uls_callsign(callsign,state,county,grid) VALUES(?,?,?,?)`)
if err != nil {
return err
}
defer stmt.Close()
sc := bufio.NewScanner(rc)
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
var n, kept int
for sc.Scan() {
if ctx.Err() != nil {
return ctx.Err()
}
n++
f := strings.Split(sc.Text(), "|")
if len(f) < 19 {
continue
}
call := strings.ToUpper(strings.TrimSpace(f[4]))
if call == "" {
continue
}
zip5 := zip5Of(f[18])
zr, ok := zipmap[zip5]
if !ok {
continue // no crosswalk entry → can't place it
}
state := strings.ToUpper(strings.TrimSpace(f[17]))
if state == "" {
state = zr.state
}
if _, err := stmt.ExecContext(ctx, call, state, zr.county, grid6(zr.lat, zr.lon)); err != nil {
return err
}
kept++
if kept%50000 == 0 {
prog("Building county database", int(math.Min(99, float64(kept)/8000)))
}
}
if err := sc.Err(); err != nil {
return fmt.Errorf("read EN.dat: %w", err)
}
if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO uls_meta(key,value) VALUES('updated_at',?)`,
time.Now().UTC().Format(time.RFC3339)); err != nil {
return err
}
if err := tx.Commit(); err != nil {
return err
}
prog("Done", 100)
return nil
}
// parseGeoNames reads US.txt out of the GeoNames zip into a zip→row map,
// keeping the first (primary) county seen for each ZIP.
func parseGeoNames(zipPath string) (map[string]zipRow, error) {
zr, err := zip.OpenReader(zipPath)
if err != nil {
return nil, err
}
defer zr.Close()
var txt *zip.File
for _, f := range zr.File {
if strings.EqualFold(filepath.Base(f.Name), "US.txt") {
txt = f
break
}
}
if txt == nil {
return nil, fmt.Errorf("US.txt not found")
}
rc, err := txt.Open()
if err != nil {
return nil, err
}
defer rc.Close()
out := make(map[string]zipRow, 45000)
sc := bufio.NewScanner(rc)
sc.Buffer(make([]byte, 0, 64*1024), 256*1024)
for sc.Scan() {
f := strings.Split(sc.Text(), "\t")
if len(f) < 11 {
continue
}
zip5 := strings.TrimSpace(f[1])
if zip5 == "" {
continue
}
if _, dup := out[zip5]; dup {
continue
}
lat := parseFloat(f[9])
lon := parseFloat(f[10])
out[zip5] = zipRow{
state: strings.ToUpper(strings.TrimSpace(f[4])),
county: strings.TrimSpace(f[5]),
lat: lat,
lon: lon,
}
}
return out, sc.Err()
}
// download streams url to dest, reporting percent when the content length is
// known and prog is non-nil.
func download(ctx context.Context, url, dest string, prog func(pct int)) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s: HTTP %d", url, resp.StatusCode)
}
f, err := os.Create(dest)
if err != nil {
return err
}
defer f.Close()
var body io.Reader = resp.Body
if prog != nil && resp.ContentLength > 0 {
body = &progReader{r: resp.Body, total: resp.ContentLength, prog: prog}
}
_, err = io.Copy(f, body)
return err
}
type progReader struct {
r io.Reader
total int64
read int64
last int
prog func(pct int)
}
func (p *progReader) Read(b []byte) (int, error) {
n, err := p.r.Read(b)
p.read += int64(n)
if pct := int(p.read * 100 / p.total); pct != p.last {
p.last = pct
p.prog(pct)
}
return n, err
}
func zip5Of(z string) string {
z = strings.TrimSpace(z)
if len(z) > 5 {
z = z[:5]
}
return z
}
func parseFloat(s string) float64 {
var v float64
fmt.Sscanf(strings.TrimSpace(s), "%g", &v)
return v
}
// grid6 converts latitude/longitude to a 6-character Maidenhead locator.
func grid6(lat, lon float64) string {
if lat == 0 && lon == 0 {
return ""
}
lon += 180
lat += 90
if lon < 0 || lon >= 360 || lat < 0 || lat >= 180 {
return ""
}
f0 := int(lon / 20)
f1 := int(lat / 10)
sq0 := int(math.Mod(lon, 20) / 2)
sq1 := int(math.Mod(lat, 10) / 1)
su0 := int(math.Mod(lon, 2) / (2.0 / 24))
su1 := int(math.Mod(lat, 1) / (1.0 / 24))
return string([]byte{
byte('A' + f0), byte('A' + f1),
byte('0' + sq0), byte('0' + sq1),
byte('a' + su0), byte('a' + su1),
})
}
+42
View File
@@ -0,0 +1,42 @@
package uls
import (
"os"
"testing"
)
func TestGrid6(t *testing.T) {
cases := []struct {
lat, lon float64
want string
}{
{38.90, -77.03, "FM18lw"}, // Washington DC
{40.71, -74.00, "FN30xr"}, // New York
{34.05, -118.24, "DM04vd"},// Los Angeles
}
for _, c := range cases {
if got := grid6(c.lat, c.lon); got[:4] != c.want[:4] {
t.Errorf("grid6(%v,%v)=%q want field/square %q", c.lat, c.lon, got, c.want[:4])
}
}
}
// TestParseGeoNames runs against the real GeoNames US.zip if present in the
// scratchpad (downloaded during development); skipped otherwise.
func TestParseGeoNames(t *testing.T) {
path := os.Getenv("GEONAMES_ZIP")
if path == "" {
t.Skip("set GEONAMES_ZIP to the US.zip path to run")
}
m, err := parseGeoNames(path)
if err != nil {
t.Fatal(err)
}
if len(m) < 30000 {
t.Fatalf("expected >30k ZIPs, got %d", len(m))
}
// A known ZIP: 20500 = The White House, DC.
if r, ok := m["20500"]; !ok || r.state != "DC" {
t.Errorf("ZIP 20500 = %+v (ok=%v)", r, ok)
}
}