chore: release v0.21.3
This commit is contained in:
+150
-7
@@ -27,10 +27,13 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -46,6 +49,14 @@ var errFCCMaintenance = errors.New("fcc uls under maintenance")
|
||||
const (
|
||||
fccAmateurURL = "https://data.fcc.gov/download/pub/uls/complete/l_amat.zip"
|
||||
geoNamesURL = "https://download.geonames.org/export/zip/US.zip"
|
||||
|
||||
// fccULSDir is the parent listing the weekly files live under. It is browsed
|
||||
// to recover from the FCC moving the file — see resolveFCCAmateurURL.
|
||||
fccULSDir = "https://data.fcc.gov/download/pub/uls/"
|
||||
|
||||
// fccAmateurFile is the weekly full-database filename, constant across the
|
||||
// FCC's directory reshuffles.
|
||||
fccAmateurFile = "l_amat.zip"
|
||||
)
|
||||
|
||||
// Location is a resolved callsign's home county + grid.
|
||||
@@ -170,10 +181,19 @@ func (s *Store) Import(ctx context.Context, tmpDir string, prog Progress) error
|
||||
return fmt.Errorf("GeoNames crosswalk is empty")
|
||||
}
|
||||
|
||||
// 2) FCC ULS full amateur database (large).
|
||||
// 2) FCC ULS full amateur database (large). The address is resolved rather
|
||||
// than assumed — the FCC moves this file between directories.
|
||||
prog("Locating FCC ULS database", 0)
|
||||
amatURL, err := resolveFCCAmateurURL(ctx)
|
||||
if err != nil {
|
||||
return err // already a user-facing explanation
|
||||
}
|
||||
if amatURL != fccAmateurURL {
|
||||
log.Printf("uls: FCC weekly file not at its usual address, using %s", amatURL)
|
||||
}
|
||||
prog("Downloading FCC ULS database", 0)
|
||||
amatPath := filepath.Join(tmpDir, "opslog_l_amat.zip")
|
||||
if err := download(ctx, fccAmateurURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
|
||||
if err := download(ctx, amatURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
|
||||
return fmt.Errorf("download FCC ULS: %w", err)
|
||||
}
|
||||
defer os.Remove(amatPath)
|
||||
@@ -323,6 +343,129 @@ func parseGeoNames(zipPath string) (map[string]zipRow, error) {
|
||||
return out, sc.Err()
|
||||
}
|
||||
|
||||
// resolveFCCAmateurURL returns a URL that actually serves the weekly amateur
|
||||
// database, working around the FCC relocating it.
|
||||
//
|
||||
// The canonical path is .../uls/complete/l_amat.zip and it is tried first. On
|
||||
// 2026-07-24 the FCC renamed that whole directory to "complete.07242026" and
|
||||
// left an empty "complete" behind, so every weekly file for every radio service
|
||||
// (not just amateur) started redirecting to a generic fcc.gov help page — the
|
||||
// county database became un-downloadable for everyone. The file itself was
|
||||
// intact the whole time, one directory across.
|
||||
//
|
||||
// Rather than hard-code that dated directory — it looks like a pre-migration
|
||||
// snapshot, and pinning it would break again the moment the FCC restores or
|
||||
// re-snapshots — we browse the parent listing and pick the most recent
|
||||
// "complete*" directory that actually holds the file. That survives the
|
||||
// canonical path coming back (tried first, so it wins), a differently-dated
|
||||
// snapshot next time, and anything else short of the file being withdrawn.
|
||||
func resolveFCCAmateurURL(ctx context.Context) (string, error) {
|
||||
if ok, _ := servesFile(ctx, fccAmateurURL); ok {
|
||||
return fccAmateurURL, nil
|
||||
}
|
||||
dirs, err := listFCCCompleteDirs(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the FCC weekly download moved and the directory listing could not be read (%w) — try again later, or check %s", err, fccULSDir)
|
||||
}
|
||||
// Newest first: the listing is alphabetical, and the dated names sort in an
|
||||
// arbitrary order (MMDDYYYY), so try them all rather than trusting the order.
|
||||
for _, d := range dirs {
|
||||
u := fccULSDir + d + fccAmateurFile
|
||||
if ok, _ := servesFile(ctx, u); ok {
|
||||
return u, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("the FCC no longer serves %s at its usual address, and no alternate directory under %s has it either — the FCC has changed its downloads; please report this", fccAmateurFile, fccULSDir)
|
||||
}
|
||||
|
||||
// servesFile reports whether url returns a real file rather than a redirect to
|
||||
// an HTML error page. The FCC answers a missing file with 302 → a help page, so
|
||||
// a plain status check on the final response is not enough: we must refuse to
|
||||
// follow the bounce and insist on a non-HTML body.
|
||||
func servesFile(ctx context.Context, url string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
if ct := resp.Header.Get("Content-Type"); strings.Contains(strings.ToLower(ct), "html") {
|
||||
return false, fmt.Errorf("served HTML, not a file")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// listFCCCompleteDirs returns the "complete*/" subdirectory names in the ULS
|
||||
// download area (e.g. "complete/", "complete.07242026/"), newest-looking last.
|
||||
func listFCCCompleteDirs(ctx context.Context) ([]string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fccULSDir, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseCompleteDirs(string(body)), nil
|
||||
}
|
||||
|
||||
// completeDirRe matches an Apache-style listing link to a "complete…" directory.
|
||||
var completeDirRe = regexp.MustCompile(`(?i)href="(complete[^"/]*/)"`)
|
||||
|
||||
// parseCompleteDirs pulls the candidate directory names out of a listing page.
|
||||
// Split out from the fetch so it can be tested against a captured listing.
|
||||
func parseCompleteDirs(html string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range completeDirRe.FindAllStringSubmatch(html, -1) {
|
||||
d := m[1]
|
||||
if d == "complete/" || seen[d] { // canonical path was already tried
|
||||
continue
|
||||
}
|
||||
seen[d] = true
|
||||
out = append(out, d)
|
||||
}
|
||||
// Dated snapshots (complete.MMDDYYYY) — prefer the most recent by date, so a
|
||||
// stale older snapshot is never picked over a fresh one.
|
||||
sort.Slice(out, func(i, j int) bool { return snapshotDate(out[i]).After(snapshotDate(out[j])) })
|
||||
return out
|
||||
}
|
||||
|
||||
var snapshotDateRe = regexp.MustCompile(`(\d{8})`)
|
||||
|
||||
// snapshotDate extracts the MMDDYYYY stamp from "complete.07242026/"; a name
|
||||
// without one sorts as the zero time (tried last).
|
||||
func snapshotDate(dir string) time.Time {
|
||||
m := snapshotDateRe.FindStringSubmatch(dir)
|
||||
if m == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
t, err := time.Parse("01022006", m[1])
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// download streams url to dest, reporting percent when the content length is
|
||||
// known and prog is non-nil.
|
||||
func download(ctx context.Context, url, dest string, prog func(pct int)) error {
|
||||
@@ -371,11 +514,11 @@ func download(ctx context.Context, url, dest string, prog func(pct int)) error {
|
||||
}
|
||||
|
||||
type progReader struct {
|
||||
r io.Reader
|
||||
total int64
|
||||
read int64
|
||||
last int
|
||||
prog func(pct int)
|
||||
r io.Reader
|
||||
total int64
|
||||
read int64
|
||||
last int
|
||||
prog func(pct int)
|
||||
}
|
||||
|
||||
func (p *progReader) Read(b []byte) (int, error) {
|
||||
|
||||
@@ -10,9 +10,9 @@ func TestGrid6(t *testing.T) {
|
||||
lat, lon float64
|
||||
want string
|
||||
}{
|
||||
{38.90, -77.03, "FM18lw"}, // Washington DC
|
||||
{40.71, -74.00, "FN30xr"}, // New York
|
||||
{34.05, -118.24, "DM04vd"},// Los Angeles
|
||||
{38.90, -77.03, "FM18lw"}, // Washington DC
|
||||
{40.71, -74.00, "FN30xr"}, // New York
|
||||
{34.05, -118.24, "DM04vd"}, // Los Angeles
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := grid6(c.lat, c.lon); got[:4] != c.want[:4] {
|
||||
@@ -40,3 +40,57 @@ func TestParseGeoNames(t *testing.T) {
|
||||
t.Errorf("ZIP 20500 = %+v (ok=%v)", r, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// The real listing captured from data.fcc.gov on 2026-07-25, the day after the
|
||||
// FCC renamed complete/ to complete.07242026/ and left an empty complete/
|
||||
// behind — which broke the county-database download for every user.
|
||||
const fccListing2026 = `<html><head><title>Index of /download/pub/uls</title></head><body>
|
||||
<h1>Index of /download/pub/uls</h1>
|
||||
<table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
|
||||
<tr><td><a href="/download/pub/">Parent Directory</a></td><td> </td><td>-</td></tr>
|
||||
<tr><td><a href="UAT/">UAT/</a></td><td>2025-04-08 19:30</td><td>-</td></tr>
|
||||
<tr><td><a href="complete.07242026/">complete.07242026/</a></td><td>2026-07-24 20:15</td><td>-</td></tr>
|
||||
<tr><td><a href="complete/">complete/</a></td><td>2026-07-25 21:15</td><td>-</td></tr>
|
||||
<tr><td><a href="daily/">daily/</a></td><td>2026-07-25 21:15</td><td>-</td></tr>
|
||||
</table></body></html>`
|
||||
|
||||
func TestParseCompleteDirs(t *testing.T) {
|
||||
got := parseCompleteDirs(fccListing2026)
|
||||
if len(got) != 1 || got[0] != "complete.07242026/" {
|
||||
t.Fatalf("got %v, want [complete.07242026/]", got)
|
||||
}
|
||||
// "complete/" is deliberately absent: it is the canonical URL, already tried
|
||||
// before the listing is consulted, and on this date it was empty.
|
||||
for _, d := range got {
|
||||
if d == "complete/" {
|
||||
t.Error("canonical complete/ should not be offered as a fallback")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Several snapshots must be tried newest-first, so a stale one is never
|
||||
// preferred over a fresh one.
|
||||
func TestParseCompleteDirsPrefersNewestSnapshot(t *testing.T) {
|
||||
html := `<a href="complete/">x</a><a href="complete.01052026/">x</a>` +
|
||||
`<a href="complete.07242026/">x</a><a href="complete.11302025/">x</a>`
|
||||
got := parseCompleteDirs(html)
|
||||
want := []string{"complete.07242026/", "complete.01052026/", "complete.11302025/"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotDate(t *testing.T) {
|
||||
if d := snapshotDate("complete.07242026/"); d.Format("2006-01-02") != "2026-07-24" {
|
||||
t.Errorf("complete.07242026/ → %s, want 2026-07-24", d.Format("2006-01-02"))
|
||||
}
|
||||
// An undated name must sort last rather than crash.
|
||||
if !snapshotDate("complete.backup/").IsZero() {
|
||||
t.Error("an undated directory should yield the zero time")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user