OpsLog/internal/uls/uls.go

// Package uls resolves a US amateur callsign to its county and grid, offline,
// from the FCC ULS licence database cross-referenced with a ZIP→county/lat-lon
// table. It backs the US Counties (USA-CA) award and county hunting: an FCC
// spot or a bare CW/SSB spot carries only a callsign, and this turns that into
// a county with no per-lookup API call.
//
// Data lives in its OWN local SQLite file (data/uls.db), never in the logbook:
// it is ~800k rows of static reference data that would only bloat the log (and
// crawl over a remote MySQL link). It is downloaded on demand, not shipped.
//
// Sources, both public and free:
//   - FCC ULS Amateur, full database: l_amat.zip → EN.dat (pipe-delimited).
//     Fields used: [4] call_sign, [17] state, [18] zip_code.
//   - GeoNames US postal codes: US.zip → US.txt (tab-delimited).
//     Fields used: [1] zip, [4] state, [5] county, [9] lat, [10] lon.
//
// The county from a ZIP is the ZIP's primary county — a ZIP can straddle a line,
// so this is ~98% right for fixed stations (rovers/portables need the from-air
// grid, which arrives separately via heard_geo). Good enough to hunt with.
package uls

import (
	"archive/zip"
	"bufio"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	_ "modernc.org/sqlite"
)

// errFCCMaintenance is raised when the FCC ULS download host bounces us to its
// maintenance page instead of serving the file (a frequent, FCC-side event).
var errFCCMaintenance = errors.New("fcc uls under maintenance")

// Default download URLs (overridable in Import for tests).
const (
	fccAmateurURL = "https://data.fcc.gov/download/pub/uls/complete/l_amat.zip"
	geoNamesURL   = "https://download.geonames.org/export/zip/US.zip"

	// fccULSDir is the parent listing the weekly files live under. It is browsed
	// to recover from the FCC moving the file — see resolveFCCAmateurURL.
	fccULSDir = "https://data.fcc.gov/download/pub/uls/"

	// fccAmateurFile is the weekly full-database filename, constant across the
	// FCC's directory reshuffles.
	fccAmateurFile = "l_amat.zip"
)

// Location is a resolved callsign's home county + grid.
type Location struct {
	State  string `json:"state"`
	County string `json:"county"` // GeoNames county name (e.g. "Middlesex")
	Grid   string `json:"grid"`   // 6-char Maidenhead from the ZIP centroid
}

// CNTY renders the ADIF "STATE,County" form for stamping a QSO's cnty field.
func (l Location) CNTY() string {
	if l.State == "" || l.County == "" {
		return ""
	}
	return l.State + "," + l.County
}

// Store owns the local uls.db connection.
type Store struct {
	db *sql.DB
	mu sync.RWMutex // guards the whole DB during a re-import (DELETE+bulk INSERT)
}

// Open opens (creating if needed) the ULS SQLite store at path.
func Open(path string) (*Store, error) {
	db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)")
	if err != nil {
		return nil, err
	}
	if _, err := db.Exec(`
		CREATE TABLE IF NOT EXISTS uls_callsign (
			callsign TEXT PRIMARY KEY,
			state    TEXT NOT NULL DEFAULT '',
			county   TEXT NOT NULL DEFAULT '',
			grid     TEXT NOT NULL DEFAULT ''
		);
		CREATE TABLE IF NOT EXISTS uls_meta (
			key   TEXT PRIMARY KEY,
			value TEXT NOT NULL DEFAULT ''
		);`); err != nil {
		db.Close()
		return nil, err
	}
	return &Store{db: db}, nil
}

func (s *Store) Close() error { return s.db.Close() }

// Count returns how many callsigns are loaded.
func (s *Store) Count() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var n int
	s.db.QueryRow(`SELECT COUNT(*) FROM uls_callsign`).Scan(&n)
	return n
}

// UpdatedAt returns when the store was last imported (zero if never).
func (s *Store) UpdatedAt() time.Time {
	s.mu.RLock()
	defer s.mu.RUnlock()
	var v string
	if err := s.db.QueryRow(`SELECT value FROM uls_meta WHERE key='updated_at'`).Scan(&v); err != nil {
		return time.Time{}
	}
	t, _ := time.Parse(time.RFC3339, v)
	return t
}

// Resolve looks up a callsign's home county + grid. ok=false if unknown or the
// store is empty.
func (s *Store) Resolve(callsign string) (Location, bool) {
	call := strings.ToUpper(strings.TrimSpace(callsign))
	if call == "" {
		return Location{}, false
	}
	s.mu.RLock()
	defer s.mu.RUnlock()
	var l Location
	err := s.db.QueryRow(`SELECT state, county, grid FROM uls_callsign WHERE callsign=?`, call).
		Scan(&l.State, &l.County, &l.Grid)
	if err != nil || l.County == "" {
		return Location{}, false
	}
	return l, true
}

// Progress reports import stages to the caller (0..100 within a stage).
type Progress func(stage string, pct int)

// zipRow is one ZIP's primary county + centroid.
type zipRow struct {
	state, county string
	lat, lon      float64
}

// Import downloads the FCC ULS + GeoNames data, rebuilds the callsign→county
// table, and records the timestamp. It replaces the table atomically: on any
// error the previous contents are left intact. tmpDir is where the (large) zips
// are streamed; "" uses the OS temp dir.
func (s *Store) Import(ctx context.Context, tmpDir string, prog Progress) error {
	if prog == nil {
		prog = func(string, int) {}
	}
	if tmpDir == "" {
		tmpDir = os.TempDir()
	}

	// 1) ZIP→county/lat-lon crosswalk (small).
	prog("Downloading ZIP crosswalk", 0)
	geoPath := filepath.Join(tmpDir, "opslog_geonames_us.zip")
	if err := download(ctx, geoNamesURL, geoPath, nil); err != nil {
		return fmt.Errorf("download GeoNames: %w", err)
	}
	defer os.Remove(geoPath)
	prog("Parsing ZIP crosswalk", 50)
	zipmap, err := parseGeoNames(geoPath)
	if err != nil {
		return fmt.Errorf("parse GeoNames: %w", err)
	}
	if len(zipmap) == 0 {
		return fmt.Errorf("GeoNames crosswalk is empty")
	}

	// 2) FCC ULS full amateur database (large). The address is resolved rather
	// than assumed — the FCC moves this file between directories.
	prog("Locating FCC ULS database", 0)
	amatURL, err := resolveFCCAmateurURL(ctx)
	if err != nil {
		return err // already a user-facing explanation
	}
	if amatURL != fccAmateurURL {
		log.Printf("uls: FCC weekly file not at its usual address, using %s", amatURL)
	}
	prog("Downloading FCC ULS database", 0)
	amatPath := filepath.Join(tmpDir, "opslog_l_amat.zip")
	if err := download(ctx, amatURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
		return fmt.Errorf("download FCC ULS: %w", err)
	}
	defer os.Remove(amatPath)

	// 3) Parse EN.dat, join the crosswalk, rebuild the table.
	prog("Building county database", 0)
	return s.rebuild(ctx, amatPath, zipmap, prog)
}

// rebuild streams EN.dat out of the FCC zip and replaces uls_callsign in one
// transaction (old data survives a failure).
func (s *Store) rebuild(ctx context.Context, amatZip string, zipmap map[string]zipRow, prog Progress) error {
	zr, err := zip.OpenReader(amatZip)
	if err != nil {
		return fmt.Errorf("open FCC zip: %w", err)
	}
	defer zr.Close()

	var en *zip.File
	for _, f := range zr.File {
		if strings.EqualFold(filepath.Base(f.Name), "EN.dat") {
			en = f
			break
		}
	}
	if en == nil {
		return fmt.Errorf("EN.dat not found in FCC zip")
	}
	rc, err := en.Open()
	if err != nil {
		return fmt.Errorf("open EN.dat: %w", err)
	}
	defer rc.Close()

	s.mu.Lock()
	defer s.mu.Unlock()

	tx, err := s.db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	if _, err := tx.ExecContext(ctx, `DELETE FROM uls_callsign`); err != nil {
		return err
	}
	stmt, err := tx.PrepareContext(ctx, `INSERT OR REPLACE INTO uls_callsign(callsign,state,county,grid) VALUES(?,?,?,?)`)
	if err != nil {
		return err
	}
	defer stmt.Close()

	sc := bufio.NewScanner(rc)
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	var n, kept int
	for sc.Scan() {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		n++
		f := strings.Split(sc.Text(), "|")
		if len(f) < 19 {
			continue
		}
		call := strings.ToUpper(strings.TrimSpace(f[4]))
		if call == "" {
			continue
		}
		zip5 := zip5Of(f[18])
		zr, ok := zipmap[zip5]
		if !ok {
			continue // no crosswalk entry → can't place it
		}
		state := strings.ToUpper(strings.TrimSpace(f[17]))
		if state == "" {
			state = zr.state
		}
		if _, err := stmt.ExecContext(ctx, call, state, zr.county, grid6(zr.lat, zr.lon)); err != nil {
			return err
		}
		kept++
		if kept%50000 == 0 {
			prog("Building county database", int(math.Min(99, float64(kept)/8000)))
		}
	}
	if err := sc.Err(); err != nil {
		return fmt.Errorf("read EN.dat: %w", err)
	}
	if _, err := tx.ExecContext(ctx, `INSERT OR REPLACE INTO uls_meta(key,value) VALUES('updated_at',?)`,
		time.Now().UTC().Format(time.RFC3339)); err != nil {
		return err
	}
	if err := tx.Commit(); err != nil {
		return err
	}
	prog("Done", 100)
	return nil
}

// parseGeoNames reads US.txt out of the GeoNames zip into a zip→row map,
// keeping the first (primary) county seen for each ZIP.
func parseGeoNames(zipPath string) (map[string]zipRow, error) {
	zr, err := zip.OpenReader(zipPath)
	if err != nil {
		return nil, err
	}
	defer zr.Close()
	var txt *zip.File
	for _, f := range zr.File {
		if strings.EqualFold(filepath.Base(f.Name), "US.txt") {
			txt = f
			break
		}
	}
	if txt == nil {
		return nil, fmt.Errorf("US.txt not found")
	}
	rc, err := txt.Open()
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	out := make(map[string]zipRow, 45000)
	sc := bufio.NewScanner(rc)
	sc.Buffer(make([]byte, 0, 64*1024), 256*1024)
	for sc.Scan() {
		f := strings.Split(sc.Text(), "\t")
		if len(f) < 11 {
			continue
		}
		zip5 := strings.TrimSpace(f[1])
		if zip5 == "" {
			continue
		}
		if _, dup := out[zip5]; dup {
			continue
		}
		lat := parseFloat(f[9])
		lon := parseFloat(f[10])
		out[zip5] = zipRow{
			state:  strings.ToUpper(strings.TrimSpace(f[4])),
			county: strings.TrimSpace(f[5]),
			lat:    lat,
			lon:    lon,
		}
	}
	return out, sc.Err()
}

// resolveFCCAmateurURL returns a URL that actually serves the weekly amateur
// database, working around the FCC relocating it.
//
// The canonical path is .../uls/complete/l_amat.zip and it is tried first. On
// 2026-07-24 the FCC renamed that whole directory to "complete.07242026" and
// left an empty "complete" behind, so every weekly file for every radio service
// (not just amateur) started redirecting to a generic fcc.gov help page — the
// county database became un-downloadable for everyone. The file itself was
// intact the whole time, one directory across.
//
// Rather than hard-code that dated directory — it looks like a pre-migration
// snapshot, and pinning it would break again the moment the FCC restores or
// re-snapshots — we browse the parent listing and pick the most recent
// "complete*" directory that actually holds the file. That survives the
// canonical path coming back (tried first, so it wins), a differently-dated
// snapshot next time, and anything else short of the file being withdrawn.
func resolveFCCAmateurURL(ctx context.Context) (string, error) {
	if ok, _ := servesFile(ctx, fccAmateurURL); ok {
		return fccAmateurURL, nil
	}
	dirs, err := listFCCCompleteDirs(ctx)
	if err != nil {
		return "", fmt.Errorf("the FCC weekly download moved and the directory listing could not be read (%w) — try again later, or check %s", err, fccULSDir)
	}
	// Newest first: the listing is alphabetical, and the dated names sort in an
	// arbitrary order (MMDDYYYY), so try them all rather than trusting the order.
	for _, d := range dirs {
		u := fccULSDir + d + fccAmateurFile
		if ok, _ := servesFile(ctx, u); ok {
			return u, nil
		}
	}
	return "", fmt.Errorf("the FCC no longer serves %s at its usual address, and no alternate directory under %s has it either — the FCC has changed its downloads; please report this", fccAmateurFile, fccULSDir)
}

// servesFile reports whether url returns a real file rather than a redirect to
// an HTML error page. The FCC answers a missing file with 302 → a help page, so
// a plain status check on the final response is not enough: we must refuse to
// follow the bounce and insist on a non-HTML body.
func servesFile(ctx context.Context, url string) (bool, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
	if err != nil {
		return false, err
	}
	client := &http.Client{
		Timeout:       30 * time.Second,
		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
	}
	resp, err := client.Do(req)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return false, fmt.Errorf("HTTP %d", resp.StatusCode)
	}
	if ct := resp.Header.Get("Content-Type"); strings.Contains(strings.ToLower(ct), "html") {
		return false, fmt.Errorf("served HTML, not a file")
	}
	return true, nil
}

// listFCCCompleteDirs returns the "complete*/" subdirectory names in the ULS
// download area (e.g. "complete/", "complete.07242026/"), newest-looking last.
func listFCCCompleteDirs(ctx context.Context) ([]string, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fccULSDir, nil)
	if err != nil {
		return nil, err
	}
	client := &http.Client{Timeout: 30 * time.Second}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return nil, err
	}
	return parseCompleteDirs(string(body)), nil
}

// completeDirRe matches an Apache-style listing link to a "complete…" directory.
var completeDirRe = regexp.MustCompile(`(?i)href="(complete[^"/]*/)"`)

// parseCompleteDirs pulls the candidate directory names out of a listing page.
// Split out from the fetch so it can be tested against a captured listing.
func parseCompleteDirs(html string) []string {
	seen := map[string]bool{}
	var out []string
	for _, m := range completeDirRe.FindAllStringSubmatch(html, -1) {
		d := m[1]
		if d == "complete/" || seen[d] { // canonical path was already tried
			continue
		}
		seen[d] = true
		out = append(out, d)
	}
	// Dated snapshots (complete.MMDDYYYY) — prefer the most recent by date, so a
	// stale older snapshot is never picked over a fresh one.
	sort.Slice(out, func(i, j int) bool { return snapshotDate(out[i]).After(snapshotDate(out[j])) })
	return out
}

var snapshotDateRe = regexp.MustCompile(`(\d{8})`)

// snapshotDate extracts the MMDDYYYY stamp from "complete.07242026/"; a name
// without one sorts as the zero time (tried last).
func snapshotDate(dir string) time.Time {
	m := snapshotDateRe.FindStringSubmatch(dir)
	if m == nil {
		return time.Time{}
	}
	t, err := time.Parse("01022006", m[1])
	if err != nil {
		return time.Time{}
	}
	return t
}

// download streams url to dest, reporting percent when the content length is
// known and prog is non-nil.
func download(ctx context.Context, url, dest string, prog func(pct int)) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	// Catch the FCC maintenance bounce BEFORE following it: data.fcc.gov redirects
	// to www.fcc.gov/system-maintenance during maintenance windows, and that page
	// then HTTP/2-stream-errors — which surfaced as a cryptic "INTERNAL_ERROR"
	// instead of a plain "try again later".
	client := &http.Client{
		CheckRedirect: func(r *http.Request, via []*http.Request) error {
			if strings.Contains(r.URL.String(), "system-maintenance") {
				return errFCCMaintenance
			}
			if len(via) >= 10 {
				return errors.New("stopped after 10 redirects")
			}
			return nil
		},
	}
	resp, err := client.Do(req)
	if err != nil {
		if errors.Is(err, errFCCMaintenance) || strings.Contains(err.Error(), "system-maintenance") {
			return fmt.Errorf("the FCC ULS download service is under maintenance (fcc.gov redirected to its maintenance page) — please try again later")
		}
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("%s: HTTP %d", url, resp.StatusCode)
	}
	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()

	var body io.Reader = resp.Body
	if prog != nil && resp.ContentLength > 0 {
		body = &progReader{r: resp.Body, total: resp.ContentLength, prog: prog}
	}
	_, err = io.Copy(f, body)
	return err
}

type progReader struct {
	r     io.Reader
	total int64
	read  int64
	last  int
	prog  func(pct int)
}

func (p *progReader) Read(b []byte) (int, error) {
	n, err := p.r.Read(b)
	p.read += int64(n)
	if pct := int(p.read * 100 / p.total); pct != p.last {
		p.last = pct
		p.prog(pct)
	}
	return n, err
}

func zip5Of(z string) string {
	z = strings.TrimSpace(z)
	if len(z) > 5 {
		z = z[:5]
	}
	return z
}

func parseFloat(s string) float64 {
	var v float64
	fmt.Sscanf(strings.TrimSpace(s), "%g", &v)
	return v
}

// grid6 converts latitude/longitude to a 6-character Maidenhead locator.
func grid6(lat, lon float64) string {
	if lat == 0 && lon == 0 {
		return ""
	}
	lon += 180
	lat += 90
	if lon < 0 || lon >= 360 || lat < 0 || lat >= 180 {
		return ""
	}
	f0 := int(lon / 20)
	f1 := int(lat / 10)
	sq0 := int(math.Mod(lon, 20) / 2)
	sq1 := int(math.Mod(lat, 10) / 1)
	su0 := int(math.Mod(lon, 2) / (2.0 / 24))
	su1 := int(math.Mod(lat, 1) / (1.0 / 24))
	return string([]byte{
		byte('A' + f0), byte('A' + f1),
		byte('0' + sq0), byte('0' + sq1),
		byte('a' + su0), byte('a' + su1),
	})
}