feat(dxped): the announced DX, judged against your own log

A DXpeditions tab holding the two feeds the DX world announces itself
on: NG3K's ADXO (structured — dates, entity, calls, bands, modes, QSL
route) and DX-World's headlines. What a logger can say that a news
reader cannot is whether the operation is worth chasing, so every
announcement is put through the SAME verdict the cluster paints on a
spot — one badge, strongest wins, dimmed when the need is only a missing
QSL — with an 'only what I need' filter. One click watches every
callsign of an operation; the news headlines are mined for callsigns so
they can be watched the same way.

Parsers ported from DXHunter's and pinned with table tests against real
feed text: the 'as PJ2/W2APF' mining, the DXCC-prefix-first
normalisation without which no spot ever matches, both ADXO date forms,
and the '160-6m' span whose unit is written once (DXHunter read that as
6m alone and lost the low end).

Watchlist membership now announces itself app-wide, on the same card as
a new version: the bindings emit watchlist:changed, so a call added from
the cluster or this tab is confirmed where the operator is actually
looking — the panel's own inline message never was.
This commit is contained in:
2026-08-31 17:51:45 +02:00
parent 629bd8d84f
commit 187ac9aa84
13 changed files with 1202 additions and 9 deletions
+546
View File
@@ -0,0 +1,546 @@
// Package dxped reads the two feeds the DX world announces itself on.
//
// NG3K's ADXO is the STRUCTURED one: an RSS item per announced operation whose
// description is a fixed, dash-separated sentence — dates, entity, callsign,
// QSL route, source, then the operators/bands/modes prose. It is what a
// DXpedition list is actually made of.
//
// DX-World's feed is NEWS: WordPress posts with a headline and an excerpt. It
// carries no structure to act on, but the headline nearly always names the
// callsign, so the calls are mined from the title and the reader can act on
// them the same way (watchlist, chase status).
//
// Both are cached: the announcements change a few times a day, the news a few
// times an hour, and neither is worth a request per screen repaint.
package dxped
import (
"context"
"encoding/xml"
"fmt"
"html"
"io"
"net/http"
"regexp"
"strings"
"sync"
"time"
)
const (
adxoURL = "https://www.ng3k.com/adxo.xml"
dxworldURL = "https://dx-world.net/feed/"
adxoTTL = 1 * time.Hour
dxworldTTL = 30 * time.Minute
)
// Activation is one announced operation, as ADXO describes it.
type Activation struct {
DXCC string `json:"dxcc"`
Callsign string `json:"callsign"` // display form; "A, B" when several
Calls []string `json:"calls"` // the individual callsigns, normalised
StartDate string `json:"start_date"`
EndDate string `json:"end_date"`
Bands []string `json:"bands"`
Modes []string `json:"modes"`
QSL string `json:"qsl"`
Operators string `json:"operators"`
Source string `json:"source"`
Link string `json:"link"`
Status string `json:"status"` // "active" | "upcoming"
}
// News is one DX-World post.
type News struct {
Title string `json:"title"`
Link string `json:"link"`
PubDate string `json:"pub_date"` // RFC3339, "" when unparseable
Excerpt string `json:"excerpt"`
Creator string `json:"creator"`
ImageURL string `json:"image_url"`
Tag string `json:"tag"` // NEWS / UPDATE / NEW ACTIVITY…
Calls []string `json:"calls"` // callsigns mined from the headline
}
// Manager holds both caches and fetches on demand.
type Manager struct {
mu sync.RWMutex
acts []Activation
actsAt time.Time
news []News
newsAt time.Time
client *http.Client
fetching sync.Mutex // one refresh at a time, whichever pane asked
}
func New() *Manager {
return &Manager{client: &http.Client{Timeout: 30 * time.Second}}
}
// Activations returns the cached announcements, refreshing when stale.
func (m *Manager) Activations(ctx context.Context) ([]Activation, error) {
m.mu.RLock()
fresh := time.Since(m.actsAt) < adxoTTL && m.acts != nil
out := m.acts
m.mu.RUnlock()
if fresh {
return out, nil
}
m.fetching.Lock()
defer m.fetching.Unlock()
// Someone else may have refreshed while we waited for the lock.
m.mu.RLock()
fresh = time.Since(m.actsAt) < adxoTTL && m.acts != nil
out = m.acts
m.mu.RUnlock()
if fresh {
return out, nil
}
acts, err := m.fetchADXO(ctx)
if err != nil {
// Stale beats empty: a feed that is down should not blank a list the
// operator was reading a minute ago.
m.mu.RLock()
defer m.mu.RUnlock()
return m.acts, err
}
m.mu.Lock()
m.acts, m.actsAt = acts, time.Now()
m.mu.Unlock()
return acts, nil
}
// News returns the cached DX-World posts, refreshing when stale.
func (m *Manager) News(ctx context.Context) ([]News, error) {
m.mu.RLock()
fresh := time.Since(m.newsAt) < dxworldTTL && m.news != nil
out := m.news
m.mu.RUnlock()
if fresh {
return out, nil
}
m.fetching.Lock()
defer m.fetching.Unlock()
m.mu.RLock()
fresh = time.Since(m.newsAt) < dxworldTTL && m.news != nil
out = m.news
m.mu.RUnlock()
if fresh {
return out, nil
}
news, err := m.fetchDXWorld(ctx)
if err != nil {
m.mu.RLock()
defer m.mu.RUnlock()
return m.news, err
}
m.mu.Lock()
m.news, m.newsAt = news, time.Now()
m.mu.Unlock()
return news, nil
}
// Invalidate drops both caches so the next read refetches.
func (m *Manager) Invalidate() {
m.mu.Lock()
m.actsAt, m.newsAt = time.Time{}, time.Time{}
m.mu.Unlock()
}
func (m *Manager) get(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "OpsLog")
resp, err := m.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http %d", resp.StatusCode)
}
return io.ReadAll(io.LimitReader(resp.Body, 8<<20))
}
// ── ADXO ────────────────────────────────────────────────────────────────
type rssItem struct {
Title string `xml:"title"`
Description string `xml:"description"`
Link string `xml:"link"`
PubDate string `xml:"pubDate"`
Creator string `xml:"creator"`
Enclosure struct {
URL string `xml:"url,attr"`
} `xml:"enclosure"`
MediaContent struct {
URL string `xml:"url,attr"`
} `xml:"content"`
}
type rssFeed struct {
Items []rssItem `xml:"channel>item"`
}
func (m *Manager) fetchADXO(ctx context.Context) ([]Activation, error) {
body, err := m.get(ctx, adxoURL)
if err != nil {
return nil, fmt.Errorf("adxo: %w", err)
}
var feed rssFeed
if err := xml.Unmarshal(body, &feed); err != nil {
return nil, fmt.Errorf("adxo: parse: %w", err)
}
now := time.Now()
out := make([]Activation, 0, len(feed.Items))
for _, it := range feed.Items {
a := parseActivation(it.Description, it.Link)
if a == nil {
continue
}
a.Status = activationStatus(a.StartDate, a.EndDate, now)
if a.Status == "ended" {
continue // the list is about what is on or coming
}
out = append(out, *a)
}
return out, nil
}
var spaceRe = regexp.MustCompile(`\s+`)
// parseActivation reads one ADXO description. Its shape is fixed and has been
// for twenty years:
//
// "Feb 17-Mar 30, 2026 -- Entity -- CALL -- QSL: route -- Source: who (date)
// -- By ops; bands; modes; notes"
func parseActivation(desc, link string) *Activation {
desc = spaceRe.ReplaceAllString(strings.NewReplacer("\n", " ", "\r", " ").Replace(desc), " ")
desc = strings.TrimSpace(html.UnescapeString(desc))
if desc == "" {
return nil
}
parts := strings.Split(desc, " -- ")
if len(parts) < 3 {
return nil
}
a := &Activation{Link: link}
a.StartDate, a.EndDate = parseDateRange(strings.TrimSpace(parts[0]))
a.DXCC = strings.TrimSpace(parts[1])
a.Callsign = strings.TrimSpace(parts[2])
for _, p := range parts[3:] {
p = strings.TrimSpace(p)
switch {
case strings.HasPrefix(p, "QSL:"):
a.QSL = strings.TrimSpace(strings.TrimPrefix(p, "QSL:"))
case strings.HasPrefix(p, "Source:"):
a.Source = strings.TrimSpace(strings.TrimPrefix(p, "Source:"))
}
}
// The tail carries "By <ops>; <bands>; <modes>; <notes>".
tail := parts[len(parts)-1]
if i := strings.Index(tail, "By "); i >= 0 {
sub := strings.Split(tail[i+3:], ";")
if len(sub) > 0 {
a.Operators = strings.TrimSpace(sub[0])
}
if len(sub) > 1 {
a.Bands = parseBands(sub[1])
}
if len(sub) > 2 {
a.Modes = parseModes(sub[2])
}
}
// The callsign field often holds only the PREFIX; the real calls hide in the
// operators prose as "W2APF as PJ2/W2APF". Prefer those when present.
if calls := callsAfterAs(a.Operators); len(calls) > 0 {
prefix := strings.ToUpper(strings.TrimSpace(parts[2]))
for i := range calls {
calls[i] = normalizeCall(calls[i], prefix)
}
a.Calls = calls
a.Callsign = strings.Join(calls, ", ")
} else if c := strings.ToUpper(a.Callsign); plausibleCall(c) {
a.Calls = []string{c}
}
return a
}
// parseDateRange handles the two forms ADXO writes: "Feb 17-Mar 30, 2026" and
// "Mar 3-20, 2026" (the month carried over).
var (
fullRangeRe = regexp.MustCompile(`(?i)(\w+ \d+)\s*-\s*(\w+ \d+),\s*(\d{4})`)
shortRangeRe = regexp.MustCompile(`(?i)(\w+) (\d+)\s*-\s*(\d+),\s*(\d{4})`)
singleDayRe = regexp.MustCompile(`(?i)(\w+ \d+),\s*(\d{4})`)
)
func parseDateRange(s string) (start, end string) {
if m := fullRangeRe.FindStringSubmatch(s); m != nil {
return m[1] + ", " + m[3], m[2] + ", " + m[3]
}
if m := shortRangeRe.FindStringSubmatch(s); m != nil {
return m[1] + " " + m[2] + ", " + m[4], m[1] + " " + m[3] + ", " + m[4]
}
if m := singleDayRe.FindStringSubmatch(s); m != nil {
return m[1] + ", " + m[2], m[1] + ", " + m[2]
}
return s, s
}
func parseADXODate(s string) (time.Time, bool) {
for _, layout := range []string{"Jan 2, 2006", "January 2, 2006"} {
if t, err := time.Parse(layout, strings.TrimSpace(s)); err == nil {
return t, true
}
}
return time.Time{}, false
}
func activationStatus(start, end string, now time.Time) string {
s, okS := parseADXODate(start)
e, okE := parseADXODate(end)
if !okS || !okE {
return "upcoming" // unreadable dates: keep it, an operator can read them
}
switch {
case now.Before(s):
return "upcoming"
// The end date is a DAY, so an operation is on until that day is over.
case now.After(e.Add(24 * time.Hour)):
return "ended"
default:
return "active"
}
}
// callsAfterAs mines "…as CALL…" out of the operators prose:
//
// "W2APF as PJ2/W2APF" → PJ2/W2APF
// "SQ2RAD as VP2EAD, M0PLX as VP2ELX" → VP2EAD, VP2ELX
var asCallRe = regexp.MustCompile(`(?i)\bas\s+([A-Z0-9]+(?:/[A-Z0-9]+)*)`)
func callsAfterAs(operators string) []string {
var out []string
seen := map[string]bool{}
for _, m := range asCallRe.FindAllStringSubmatch(operators, -1) {
c := strings.ToUpper(strings.TrimSpace(m[1]))
if !plausibleCall(c) || seen[c] {
continue
}
seen[c] = true
out = append(out, c)
}
return out
}
// normalizeCall puts the DXCC prefix first: a cluster spot says JD1/JG8NQJ, and
// matching the log against JG8NQJ/JD1 would find nothing.
func normalizeCall(call, dxccPrefix string) string {
if dxccPrefix == "" || !strings.Contains(call, "/") {
return call
}
left, right, _ := strings.Cut(call, "/")
if strings.HasPrefix(right, dxccPrefix) && !strings.HasPrefix(left, dxccPrefix) {
return right + "/" + left
}
return call
}
var (
// A span carries its unit only once — "160-6m" is the commonest way ADXO
// states coverage, and reading it as "6m" alone loses the whole low end.
bandRe = regexp.MustCompile(`(?i)\b(?:(\d{1,4})\s*-\s*)?(\d{1,4})\s*(m|cm)\b`)
// The modes ADXO actually writes. A list beats a pattern here: "FT8" and
// "SSB" have no shape in common, and inventing one invites "QSL" as a mode.
knownModes = []string{"SSB", "CW", "FT8", "FT4", "RTTY", "PSK", "SSTV", "AM", "FM", "JT65", "JS8", "Q65", "MSK144", "DIGI", "DATA"}
)
func parseBands(s string) []string {
var out []string
seen := map[string]bool{}
add := func(num, unit string) {
if num == "" {
return
}
b := strings.ToLower(num + unit)
if seen[b] {
return
}
seen[b] = true
out = append(out, b)
}
// A span keeps only its endpoints: ADXO states coverage in prose, and the
// two ends are the part it gives reliably.
for _, m := range bandRe.FindAllStringSubmatch(s, -1) {
add(m[1], m[3])
add(m[2], m[3])
}
return out
}
func parseModes(s string) []string {
up := strings.ToUpper(s)
var out []string
seen := map[string]bool{}
for _, mode := range knownModes {
if seen[mode] {
continue
}
if regexp.MustCompile(`\b` + mode + `\b`).MatchString(up) {
seen[mode] = true
out = append(out, mode)
}
}
return out
}
// ── DX-World ────────────────────────────────────────────────────────────
var (
htmlTagRe = regexp.MustCompile(`<[^>]+>`)
tagPrefixRe = regexp.MustCompile(`(?i)^\s*\[([^\]]+)\]\s*`)
)
func stripHTML(s string) string {
s = htmlTagRe.ReplaceAllString(s, " ")
s = html.UnescapeString(s)
return strings.TrimSpace(spaceRe.ReplaceAllString(s, " "))
}
func (m *Manager) fetchDXWorld(ctx context.Context) ([]News, error) {
body, err := m.get(ctx, dxworldURL)
if err != nil {
return nil, fmt.Errorf("dx-world: %w", err)
}
var feed rssFeed
if err := xml.Unmarshal(body, &feed); err != nil {
return nil, fmt.Errorf("dx-world: parse: %w", err)
}
out := make([]News, 0, len(feed.Items))
for _, it := range feed.Items {
title := stripHTML(it.Title)
tag, title := splitTag(title)
n := News{
Title: title,
Link: strings.TrimSpace(it.Link),
Creator: strings.TrimSpace(it.Creator),
Tag: tag,
Calls: callsInHeadline(title),
}
if t, err := time.Parse(time.RFC1123Z, strings.TrimSpace(it.PubDate)); err == nil {
n.PubDate = t.UTC().Format(time.RFC3339)
} else if t, err := time.Parse(time.RFC1123, strings.TrimSpace(it.PubDate)); err == nil {
n.PubDate = t.UTC().Format(time.RFC3339)
}
ex := stripHTML(it.Description)
if t2, rest := splitTag(ex); t2 != "" {
if n.Tag == "" {
n.Tag = t2
}
ex = rest
}
n.Excerpt = truncateRunes(ex, 400)
if u := strings.TrimSpace(it.Enclosure.URL); u != "" {
n.ImageURL = u
} else if u := strings.TrimSpace(it.MediaContent.URL); u != "" {
n.ImageURL = u
}
out = append(out, n)
}
return out, nil
}
// splitTag pulls the leading "[UPDATE]" DX-World puts on most posts.
func splitTag(s string) (tag, rest string) {
m := tagPrefixRe.FindStringSubmatchIndex(s)
if m == nil {
return "", s
}
tag = strings.ToUpper(strings.TrimSpace(s[m[2]:m[3]]))
rest = strings.TrimSpace(s[m[1]:])
rest = strings.TrimPrefix(strings.TrimPrefix(rest, " "), "- ")
return tag, strings.TrimSpace(rest)
}
func truncateRunes(s string, max int) string {
r := []rune(s)
if len(r) <= max {
return s
}
return strings.TrimSpace(string(r[:max])) + "…"
}
// callsInHeadline mines callsigns out of a news headline — "3B7M, St Brandon"
// or "TX5S team lands". The reader can then chase or watch them, which is the
// whole reason a news feed sits next to the announcements.
func callsInHeadline(title string) []string {
var out []string
seen := map[string]bool{}
for _, tok := range strings.FieldsFunc(title, func(r rune) bool {
return !(r == '/' || (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z'))
}) {
c := strings.ToUpper(tok)
if !plausibleCall(c) || seen[c] {
continue
}
seen[c] = true
out = append(out, c)
}
return out
}
var (
bandTokenRe = regexp.MustCompile(`^\d{1,4}(M|CM)$`)
// Jargon shaped exactly like a callsign. Every one of these was seen in a
// real headline before it earned its place here.
notACall = map[string]bool{
"FT8": true, "FT4": true, "JT65": true, "JT9": true, "JS8": true, "Q65": true,
"MSK144": true, "PSK31": true, "SSTV": true, "OQRS": true, "LOTW": true,
"IOTA": true, "SOTA": true, "POTA": true, "WWFF": true, "DXCC": true,
"CQWW": true, "CQWPX": true, "ARRL": true, "3D": true, "4K": true,
}
)
// plausibleCall keeps tokens shaped like an amateur callsign: 312 characters
// of AZ/09 (with optional /prefix or /suffix), at least one letter AND one
// digit, and a letter somewhere after the first digit — which is what separates
// a callsign from a band or a year.
func plausibleCall(s string) bool {
s = strings.ToUpper(strings.TrimSpace(s))
if len(s) < 3 || len(s) > 12 || notACall[s] || bandTokenRe.MatchString(s) {
return false
}
// Judge the longest part — the real call in "PJ2/W2APF" either way.
base := s
if strings.Contains(s, "/") {
base = ""
for _, p := range strings.Split(s, "/") {
if len(p) > len(base) {
base = p
}
}
}
var hasLetter, hasDigit, letterAfterDigit bool
seenDigit := false
for _, r := range base {
switch {
case r >= 'A' && r <= 'Z':
hasLetter = true
if seenDigit {
letterAfterDigit = true
}
case r >= '0' && r <= '9':
hasDigit = true
seenDigit = true
default:
return false
}
}
return hasLetter && hasDigit && letterAfterDigit
}