404 lines
10 KiB
Go
404 lines
10 KiB
Go
// Package uls resolves a US amateur callsign to its county and grid, offline,
|
|
// from the FCC ULS licence database cross-referenced with a ZIP→county/lat-lon
|
|
// table. It backs the US Counties (USA-CA) award and county hunting: an FCC
|
|
// spot or a bare CW/SSB spot carries only a callsign, and this turns that into
|
|
// a county with no per-lookup API call.
|
|
//
|
|
// Data lives in its OWN local SQLite file (data/uls.db), never in the logbook:
|
|
// it is ~800k rows of static reference data that would only bloat the log (and
|
|
// crawl over a remote MySQL link). It is downloaded on demand, not shipped.
|
|
//
|
|
// Sources, both public and free:
|
|
// - FCC ULS Amateur, full database: l_amat.zip → EN.dat (pipe-delimited).
|
|
// Fields used: [4] call_sign, [17] state, [18] zip_code.
|
|
// - GeoNames US postal codes: US.zip → US.txt (tab-delimited).
|
|
// Fields used: [1] zip, [4] state, [5] county, [9] lat, [10] lon.
|
|
//
|
|
// The county from a ZIP is the ZIP's primary county — a ZIP can straddle a line,
|
|
// so this is ~98% right for fixed stations (rovers/portables need the from-air
|
|
// grid, which arrives separately via heard_geo). Good enough to hunt with.
|
|
package uls
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bufio"
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// Default download URLs (overridable in Import for tests).
|
|
const (
|
|
fccAmateurURL = "https://data.fcc.gov/download/pub/uls/complete/l_amat.zip"
|
|
geoNamesURL = "https://download.geonames.org/export/zip/US.zip"
|
|
)
|
|
|
|
// Location is a resolved callsign's home county + grid.
|
|
type Location struct {
|
|
State string `json:"state"`
|
|
County string `json:"county"` // GeoNames county name (e.g. "Middlesex")
|
|
Grid string `json:"grid"` // 6-char Maidenhead from the ZIP centroid
|
|
}
|
|
|
|
// CNTY renders the ADIF "STATE,County" form for stamping a QSO's cnty field.
|
|
func (l Location) CNTY() string {
|
|
if l.State == "" || l.County == "" {
|
|
return ""
|
|
}
|
|
return l.State + "," + l.County
|
|
}
|
|
|
|
// Store owns the local uls.db connection.
|
|
type Store struct {
|
|
db *sql.DB
|
|
mu sync.RWMutex // guards the whole DB during a re-import (DELETE+bulk INSERT)
|
|
}
|
|
|
|
// Open opens (creating if needed) the ULS SQLite store at path.
|
|
func Open(path string) (*Store, error) {
|
|
db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS uls_callsign (
|
|
callsign TEXT PRIMARY KEY,
|
|
state TEXT NOT NULL DEFAULT '',
|
|
county TEXT NOT NULL DEFAULT '',
|
|
grid TEXT NOT NULL DEFAULT ''
|
|
);
|
|
CREATE TABLE IF NOT EXISTS uls_meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL DEFAULT ''
|
|
);`); err != nil {
|
|
db.Close()
|
|
return nil, err
|
|
}
|
|
return &Store{db: db}, nil
|
|
}
|
|
|
|
func (s *Store) Close() error { return s.db.Close() }
|
|
|
|
// Count returns how many callsigns are loaded.
|
|
func (s *Store) Count() int {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
var n int
|
|
s.db.QueryRow(`SELECT COUNT(*) FROM uls_callsign`).Scan(&n)
|
|
return n
|
|
}
|
|
|
|
// UpdatedAt returns when the store was last imported (zero if never).
|
|
func (s *Store) UpdatedAt() time.Time {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
var v string
|
|
if err := s.db.QueryRow(`SELECT value FROM uls_meta WHERE key='updated_at'`).Scan(&v); err != nil {
|
|
return time.Time{}
|
|
}
|
|
t, _ := time.Parse(time.RFC3339, v)
|
|
return t
|
|
}
|
|
|
|
// Resolve looks up a callsign's home county + grid. ok=false if unknown or the
|
|
// store is empty.
|
|
func (s *Store) Resolve(callsign string) (Location, bool) {
|
|
call := strings.ToUpper(strings.TrimSpace(callsign))
|
|
if call == "" {
|
|
return Location{}, false
|
|
}
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
var l Location
|
|
err := s.db.QueryRow(`SELECT state, county, grid FROM uls_callsign WHERE callsign=?`, call).
|
|
Scan(&l.State, &l.County, &l.Grid)
|
|
if err != nil || l.County == "" {
|
|
return Location{}, false
|
|
}
|
|
return l, true
|
|
}
|
|
|
|
// Progress reports import stages to the caller (0..100 within a stage).
|
|
type Progress func(stage string, pct int)
|
|
|
|
// zipRow is one ZIP's primary county + centroid.
|
|
type zipRow struct {
|
|
state, county string
|
|
lat, lon float64
|
|
}
|
|
|
|
// Import downloads the FCC ULS + GeoNames data, rebuilds the callsign→county
|
|
// table, and records the timestamp. It replaces the table atomically: on any
|
|
// error the previous contents are left intact. tmpDir is where the (large) zips
|
|
// are streamed; "" uses the OS temp dir.
|
|
func (s *Store) Import(ctx context.Context, tmpDir string, prog Progress) error {
|
|
if prog == nil {
|
|
prog = func(string, int) {}
|
|
}
|
|
if tmpDir == "" {
|
|
tmpDir = os.TempDir()
|
|
}
|
|
|
|
// 1) ZIP→county/lat-lon crosswalk (small).
|
|
prog("Downloading ZIP crosswalk", 0)
|
|
geoPath := filepath.Join(tmpDir, "opslog_geonames_us.zip")
|
|
if err := download(ctx, geoNamesURL, geoPath, nil); err != nil {
|
|
return fmt.Errorf("download GeoNames: %w", err)
|
|
}
|
|
defer os.Remove(geoPath)
|
|
prog("Parsing ZIP crosswalk", 50)
|
|
zipmap, err := parseGeoNames(geoPath)
|
|
if err != nil {
|
|
return fmt.Errorf("parse GeoNames: %w", err)
|
|
}
|
|
if len(zipmap) == 0 {
|
|
return fmt.Errorf("GeoNames crosswalk is empty")
|
|
}
|
|
|
|
// 2) FCC ULS full amateur database (large).
|
|
prog("Downloading FCC ULS database", 0)
|
|
amatPath := filepath.Join(tmpDir, "opslog_l_amat.zip")
|
|
if err := download(ctx, fccAmateurURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
|
|
return fmt.Errorf("download FCC ULS: %w", err)
|
|
}
|
|
defer os.Remove(amatPath)
|
|
|
|
// 3) Parse EN.dat, join the crosswalk, rebuild the table.
|
|
prog("Building county database", 0)
|
|
return s.rebuild(ctx, amatPath, zipmap, prog)
|
|
}
|
|
|
|
// rebuild streams EN.dat out of the FCC zip and replaces uls_callsign in one
|
|
// transaction (old data survives a failure).
|
|
func (s *Store) rebuild(ctx context.Context, amatZip string, zipmap map[string]zipRow, prog Progress) error {
|
|
zr, err := zip.OpenReader(amatZip)
|
|
if err != nil {
|
|
return fmt.Errorf("open FCC zip: %w", err)
|
|
}
|
|
defer zr.Close()
|
|
|
|
var en *zip.File
|
|
for _, f := range zr.File {
|
|
if strings.EqualFold(filepath.Base(f.Name), "EN.dat") {
|
|
en = f
|
|
break
|
|
}
|
|
}
|
|
if en == nil {
|
|
return fmt.Errorf("EN.dat not found in FCC zip")
|
|
}
|
|
rc, err := en.Open()
|
|
if err != nil {
|
|
return fmt.Errorf("open EN.dat: %w", err)
|
|
}
|
|
defer rc.Close()
|
|
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if _, err := tx.ExecContext(ctx, `DELETE FROM uls_callsign`); err != nil {
|
|
return err
|
|
}
|
|
stmt, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO uls_callsign(callsign,state,county,grid) VALUES(?,?,?,?)`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer stmt.Close()
|
|
|
|
sc := bufio.NewScanner(rc)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
|
var n, kept int
|
|
for sc.Scan() {
|
|
if ctx.Err() != nil {
|
|
return ctx.Err()
|
|
}
|
|
n++
|
|
f := strings.Split(sc.Text(), "|")
|
|
if len(f) < 19 {
|
|
continue
|
|
}
|
|
call := strings.ToUpper(strings.TrimSpace(f[4]))
|
|
if call == "" {
|
|
continue
|
|
}
|
|
zip5 := zip5Of(f[18])
|
|
zr, ok := zipmap[zip5]
|
|
if !ok {
|
|
continue // no crosswalk entry → can't place it
|
|
}
|
|
state := strings.ToUpper(strings.TrimSpace(f[17]))
|
|
if state == "" {
|
|
state = zr.state
|
|
}
|
|
if _, err := stmt.ExecContext(ctx, call, state, zr.county, grid6(zr.lat, zr.lon)); err != nil {
|
|
return err
|
|
}
|
|
kept++
|
|
if kept%50000 == 0 {
|
|
prog("Building county database", int(math.Min(99, float64(kept)/8000)))
|
|
}
|
|
}
|
|
if err := sc.Err(); err != nil {
|
|
return fmt.Errorf("read EN.dat: %w", err)
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO uls_meta(key,value) VALUES('updated_at',?)`,
|
|
time.Now().UTC().Format(time.RFC3339)); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
prog("Done", 100)
|
|
return nil
|
|
}
|
|
|
|
// parseGeoNames reads US.txt out of the GeoNames zip into a zip→row map,
|
|
// keeping the first (primary) county seen for each ZIP.
|
|
func parseGeoNames(zipPath string) (map[string]zipRow, error) {
|
|
zr, err := zip.OpenReader(zipPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer zr.Close()
|
|
var txt *zip.File
|
|
for _, f := range zr.File {
|
|
if strings.EqualFold(filepath.Base(f.Name), "US.txt") {
|
|
txt = f
|
|
break
|
|
}
|
|
}
|
|
if txt == nil {
|
|
return nil, fmt.Errorf("US.txt not found")
|
|
}
|
|
rc, err := txt.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rc.Close()
|
|
|
|
out := make(map[string]zipRow, 45000)
|
|
sc := bufio.NewScanner(rc)
|
|
sc.Buffer(make([]byte, 0, 64*1024), 256*1024)
|
|
for sc.Scan() {
|
|
f := strings.Split(sc.Text(), "\t")
|
|
if len(f) < 11 {
|
|
continue
|
|
}
|
|
zip5 := strings.TrimSpace(f[1])
|
|
if zip5 == "" {
|
|
continue
|
|
}
|
|
if _, dup := out[zip5]; dup {
|
|
continue
|
|
}
|
|
lat := parseFloat(f[9])
|
|
lon := parseFloat(f[10])
|
|
out[zip5] = zipRow{
|
|
state: strings.ToUpper(strings.TrimSpace(f[4])),
|
|
county: strings.TrimSpace(f[5]),
|
|
lat: lat,
|
|
lon: lon,
|
|
}
|
|
}
|
|
return out, sc.Err()
|
|
}
|
|
|
|
// download streams url to dest, reporting percent when the content length is
|
|
// known and prog is non-nil.
|
|
func download(ctx context.Context, url, dest string, prog func(pct int)) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("%s: HTTP %d", url, resp.StatusCode)
|
|
}
|
|
f, err := os.Create(dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
var body io.Reader = resp.Body
|
|
if prog != nil && resp.ContentLength > 0 {
|
|
body = &progReader{r: resp.Body, total: resp.ContentLength, prog: prog}
|
|
}
|
|
_, err = io.Copy(f, body)
|
|
return err
|
|
}
|
|
|
|
type progReader struct {
|
|
r io.Reader
|
|
total int64
|
|
read int64
|
|
last int
|
|
prog func(pct int)
|
|
}
|
|
|
|
func (p *progReader) Read(b []byte) (int, error) {
|
|
n, err := p.r.Read(b)
|
|
p.read += int64(n)
|
|
if pct := int(p.read * 100 / p.total); pct != p.last {
|
|
p.last = pct
|
|
p.prog(pct)
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func zip5Of(z string) string {
|
|
z = strings.TrimSpace(z)
|
|
if len(z) > 5 {
|
|
z = z[:5]
|
|
}
|
|
return z
|
|
}
|
|
|
|
func parseFloat(s string) float64 {
|
|
var v float64
|
|
fmt.Sscanf(strings.TrimSpace(s), "%g", &v)
|
|
return v
|
|
}
|
|
|
|
// grid6 converts latitude/longitude to a 6-character Maidenhead locator.
|
|
func grid6(lat, lon float64) string {
|
|
if lat == 0 && lon == 0 {
|
|
return ""
|
|
}
|
|
lon += 180
|
|
lat += 90
|
|
if lon < 0 || lon >= 360 || lat < 0 || lat >= 180 {
|
|
return ""
|
|
}
|
|
f0 := int(lon / 20)
|
|
f1 := int(lat / 10)
|
|
sq0 := int(math.Mod(lon, 20) / 2)
|
|
sq1 := int(math.Mod(lat, 10) / 1)
|
|
su0 := int(math.Mod(lon, 2) / (2.0 / 24))
|
|
su1 := int(math.Mod(lat, 1) / (1.0 / 24))
|
|
return string([]byte{
|
|
byte('A' + f0), byte('A' + f1),
|
|
byte('0' + sq0), byte('0' + sq1),
|
|
byte('a' + su0), byte('a' + su1),
|
|
})
|
|
}
|