rouggy/OpsLog
0
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

chore: release v0.21.3

Browse Source
This commit is contained in:
rouggy
2026-07-26 16:57:19 +02:00
parent 4fd70f6a9d
commit 91b5af1c7b
46 changed files with 2491 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/uls/uls.go
+150 -7
View File
@@ -27,10 +27,13 @@ import (
"errors"
"fmt"
"io"
"log"
"math"
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"time"
@@ -46,6 +49,14 @@ var errFCCMaintenance = errors.New("fcc uls under maintenance")
const (
fccAmateurURL = "https://data.fcc.gov/download/pub/uls/complete/l_amat.zip"
geoNamesURL = "https://download.geonames.org/export/zip/US.zip"
// fccULSDir is the parent listing the weekly files live under. It is browsed
// to recover from the FCC moving the file — see resolveFCCAmateurURL.
fccULSDir = "https://data.fcc.gov/download/pub/uls/"
// fccAmateurFile is the weekly full-database filename, constant across the
// FCC's directory reshuffles.
fccAmateurFile = "l_amat.zip"
)
// Location is a resolved callsign's home county + grid.
@@ -170,10 +181,19 @@ func (s *Store) Import(ctx context.Context, tmpDir string, prog Progress) error
return fmt.Errorf("GeoNames crosswalk is empty")
}
// 2) FCC ULS full amateur database (large).
// 2) FCC ULS full amateur database (large). The address is resolved rather
// than assumed — the FCC moves this file between directories.
prog("Locating FCC ULS database", 0)
amatURL, err := resolveFCCAmateurURL(ctx)
if err != nil {
return err // already a user-facing explanation
}
if amatURL != fccAmateurURL {
log.Printf("uls: FCC weekly file not at its usual address, using %s", amatURL)
}
prog("Downloading FCC ULS database", 0)
amatPath := filepath.Join(tmpDir, "opslog_l_amat.zip")
if err := download(ctx, fccAmateurURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
if err := download(ctx, amatURL, amatPath, func(pct int) { prog("Downloading FCC ULS database", pct) }); err != nil {
return fmt.Errorf("download FCC ULS: %w", err)
}
defer os.Remove(amatPath)
@@ -323,6 +343,129 @@ func parseGeoNames(zipPath string) (map[string]zipRow, error) {
return out, sc.Err()
}
// resolveFCCAmateurURL returns a URL that actually serves the weekly amateur
// database, working around the FCC relocating it.
//
// The canonical path is .../uls/complete/l_amat.zip and it is tried first. On
// 2026-07-24 the FCC renamed that whole directory to "complete.07242026" and
// left an empty "complete" behind, so every weekly file for every radio service
// (not just amateur) started redirecting to a generic fcc.gov help page — the
// county database became un-downloadable for everyone. The file itself was
// intact the whole time, one directory across.
//
// Rather than hard-code that dated directory — it looks like a pre-migration
// snapshot, and pinning it would break again the moment the FCC restores or
// re-snapshots — we browse the parent listing and pick the most recent
// "complete*" directory that actually holds the file. That survives the
// canonical path coming back (tried first, so it wins), a differently-dated
// snapshot next time, and anything else short of the file being withdrawn.
func resolveFCCAmateurURL(ctx context.Context) (string, error) {
if ok, _ := servesFile(ctx, fccAmateurURL); ok {
return fccAmateurURL, nil
}
dirs, err := listFCCCompleteDirs(ctx)
if err != nil {
return "", fmt.Errorf("the FCC weekly download moved and the directory listing could not be read (%w) — try again later, or check %s", err, fccULSDir)
}
// Newest first: the listing is alphabetical, and the dated names sort in an
// arbitrary order (MMDDYYYY), so try them all rather than trusting the order.
for _, d := range dirs {
u := fccULSDir + d + fccAmateurFile
if ok, _ := servesFile(ctx, u); ok {
return u, nil
}
}
return "", fmt.Errorf("the FCC no longer serves %s at its usual address, and no alternate directory under %s has it either — the FCC has changed its downloads; please report this", fccAmateurFile, fccULSDir)
}
// servesFile reports whether url returns a real file rather than a redirect to
// an HTML error page. The FCC answers a missing file with 302 → a help page, so
// a plain status check on the final response is not enough: we must refuse to
// follow the bounce and insist on a non-HTML body.
func servesFile(ctx context.Context, url string) (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return false, err
}
client := &http.Client{
Timeout: 30 * time.Second,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
resp, err := client.Do(req)
if err != nil {
return false, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Errorf("HTTP %d", resp.StatusCode)
}
if ct := resp.Header.Get("Content-Type"); strings.Contains(strings.ToLower(ct), "html") {
return false, fmt.Errorf("served HTML, not a file")
}
return true, nil
}
// listFCCCompleteDirs returns the "complete*/" subdirectory names in the ULS
// download area (e.g. "complete/", "complete.07242026/"), newest-looking last.
func listFCCCompleteDirs(ctx context.Context) ([]string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fccULSDir, nil)
if err != nil {
return nil, err
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return nil, err
}
return parseCompleteDirs(string(body)), nil
}
// completeDirRe matches an Apache-style listing link to a "complete…" directory.
var completeDirRe = regexp.MustCompile(`(?i)href="(complete[^"/]*/)"`)
// parseCompleteDirs pulls the candidate directory names out of a listing page.
// Split out from the fetch so it can be tested against a captured listing.
func parseCompleteDirs(html string) []string {
seen := map[string]bool{}
var out []string
for _, m := range completeDirRe.FindAllStringSubmatch(html, -1) {
d := m[1]
if d == "complete/" || seen[d] { // canonical path was already tried
continue
}
seen[d] = true
out = append(out, d)
}
// Dated snapshots (complete.MMDDYYYY) — prefer the most recent by date, so a
// stale older snapshot is never picked over a fresh one.
sort.Slice(out, func(i, j int) bool { return snapshotDate(out[i]).After(snapshotDate(out[j])) })
return out
}
var snapshotDateRe = regexp.MustCompile(`(\d{8})`)
// snapshotDate extracts the MMDDYYYY stamp from "complete.07242026/"; a name
// without one sorts as the zero time (tried last).
func snapshotDate(dir string) time.Time {
m := snapshotDateRe.FindStringSubmatch(dir)
if m == nil {
return time.Time{}
}
t, err := time.Parse("01022006", m[1])
if err != nil {
return time.Time{}
}
return t
}
// download streams url to dest, reporting percent when the content length is
// known and prog is non-nil.
func download(ctx context.Context, url, dest string, prog func(pct int)) error {
@@ -371,11 +514,11 @@ func download(ctx context.Context, url, dest string, prog func(pct int)) error {
}
type progReader struct {
r io.Reader
total int64
read int64
last int
prog func(pct int)
r io.Reader
total int64
read int64
last int
prog func(pct int)
}
func (p *progReader) Read(b []byte) (int, error) {