OpsLog/internal/extsvc/lotw.go

package extsvc

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// lotwReportURL is LoTW's confirmation-report endpoint. It returns an ADIF
// document of the user's QSOs (optionally only confirmed ones).
const lotwReportURL = "https://lotw.arrl.org/lotwuser/lotwreport.adi"

const (
	// How long LoTW may take to START answering. It builds the whole report
	// before sending anything, so this is the slow part of a big account.
	lotwHeaderTimeout = 10 * time.Minute
	// How long the transfer may stall once it HAS started. A download that has
	// not moved in this long is not slow, it is dead — and saying so beats a
	// progress window that sits at "working" until someone gives up.
	lotwIdleTimeout = 2 * time.Minute
	lotwMaxBytes    = 256 * 1024 * 1024
)

// readWithProgress reads the body in chunks, reporting the running total and
// failing fast on a stall.
//
// Reported as it arrives rather than at the end: an 18 MB report over a slow
// link is minutes of silence otherwise, which is indistinguishable from a hang —
// and that is exactly what operators were reporting.
func say(note func(string), msg string) {
	if note != nil {
		note(msg)
	}
	LogSink("%s", msg)
}

func readWithProgress(ctx context.Context, r io.Reader, note func(string)) ([]byte, error) {
	var (
		out   []byte
		total int64
		last  = time.Now()
		buf   = make([]byte, 64*1024)
		next  = int64(256 * 1024) // first report early — proof it is moving
	)
	for {
		if err := ctx.Err(); err != nil {
			return nil, err
		}
		n, err := r.Read(buf)
		if n > 0 {
			out = append(out, buf[:n]...)
			total += int64(n)
			last = time.Now()
			if total >= next {
				say(note, fmt.Sprintf("  … %.1f MB received", float64(total)/(1024*1024)))
				next = total + 512*1024
			}
			if total >= lotwMaxBytes {
				return out, nil
			}
		}
		if err == io.EOF {
			return out, nil
		}
		if err != nil {
			return nil, err
		}
		if time.Since(last) > lotwIdleTimeout {
			return nil, fmt.Errorf("the transfer stalled after %d KB — LoTW stopped sending", total/1024)
		}
	}
}

// DownloadLoTWConfirmations fetches confirmed QSOs from LoTW as ADIF text.
// Uses the LoTW *website* login (Username/Password), not the TQSL cert. When
// since is non-empty (YYYY-MM-DD) only confirmations received since then are
// returned — used for incremental "Last download" updates. When ownCall is
// non-empty, only confirmations for that station callsign are returned (an
// LoTW account holds every call you operate — F4BPO, F4BPO/P, TM2Q — so this
// scopes the pull to the active profile's call).
func DownloadLoTWConfirmations(ctx context.Context, client *http.Client, cfg ServiceConfig, since, ownCall string, detail bool, note func(string)) (string, error) {
	user := strings.TrimSpace(cfg.Username)
	if user == "" || cfg.Password == "" {
		return "", fmt.Errorf("lotw: website login (username/password) not set")
	}
	q := url.Values{}
	q.Set("login", user)
	q.Set("password", cfg.Password)
	q.Set("qso_query", "1")
	q.Set("qso_qsl", "yes") // only QSLed (confirmed) records
	// qso_qsldetail is what LoTW charges for: it adds the QSL date and the
	// station's own DXCC / grid / state / county to every record, and takes an
	// order of magnitude longer to build — a report that arrives in two minutes
	// without it takes twenty with it, measured on the same account.
	//
	// What we actually need to mark a confirmation is call, date, band and mode.
	// The rest is worth its price only when the download is also ADDING the QSOs
	// it cannot find, which is the one case where the extra fields are the only
	// source for them.
	if detail {
		q.Set("qso_qsldetail", "yes")
	}
	if c := strings.TrimSpace(ownCall); c != "" {
		q.Set("qso_owncall", c) // restrict to this station callsign
	}
	// qso_qslsince is ALWAYS sent, even for "everything".
	//
	// Left out, LoTW does not answer "all confirmations" — it answers with a
	// handful of recent ones, which arrives as a 200 and a valid ADIF and reads
	// as a successful download of a nearly empty account. Asking from a date
	// older than the service itself is the only way to mean "all".
	sinceDate := strings.TrimSpace(since)
	if sinceDate == "" {
		sinceDate = "1945-11-15" // older than any QSO LoTW will accept
	}
	q.Set("qso_qslsince", sinceDate)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, lotwReportURL+"?"+q.Encode(), nil)
	if err != nil {
		return "", fmt.Errorf("lotw: build request: %w", err)
	}
	// Named, because LoTW's front end throttles unidentified clients harder than
	// it throttles known ones, and an operator reporting a 503 deserves a request
	// that says who is asking.
	req.Header.Set("User-Agent", "OpsLog")
	if client == nil {
		// NO overall deadline. A full account is tens of megabytes and LoTW spends
		// minutes building it before the first byte; a total timeout turns a slow
		// but healthy download into "context deadline exceeded", and a longer one
		// turns a dead connection into a window that says "working" for twenty
		// minutes. What matters is not how long it takes but whether it is still
		// moving — see the idle watchdog below.
		client = &http.Client{
			Transport: &http.Transport{
				Proxy:                 http.ProxyFromEnvironment,
				ResponseHeaderTimeout: lotwHeaderTimeout,
				TLSHandshakeTimeout:   30 * time.Second,
			},
		}
	}
	// LoTW answers 503 when it is busy, which for a report covering more than a
	// few days is often — other loggers get the same answer and simply ask
	// again. Three tries, spaced, and each one said out loud: an operator whose
	// download takes four minutes because the ARRL is loaded should be able to
	// see that rather than guess it.
	// LoTW sends nothing at all until the whole report is built — minutes for a
	// large account. That silence was the entire complaint: a window saying
	// "working" with no way to tell a busy server from a dead one. Count it out
	// loud until the first byte.
	beat := make(chan struct{})
	go func() {
		start := time.Now()
		tick := time.NewTicker(15 * time.Second)
		defer tick.Stop()
		for {
			select {
			case <-beat:
				return
			case <-ctx.Done():
				return
			case <-tick.C:
				say(note, fmt.Sprintf("  … still waiting for LoTW to build the report (%.0f s)", time.Since(start).Seconds()))
			}
		}
	}()
	// Stopped where the WAIT ends, not where the function does: deferred, the
	// heartbeat went on counting between the megabyte lines and read as if the
	// report were still being built while it was already arriving.
	stopBeat := sync.OnceFunc(func() { close(beat) })
	defer stopBeat()

	var resp *http.Response
	for attempt := 1; ; attempt++ {
		resp, err = client.Do(req) //nolint:bodyclose // closed below or in the retry
		if err == nil && resp.StatusCode != http.StatusServiceUnavailable &&
			resp.StatusCode != http.StatusBadGateway && resp.StatusCode != http.StatusGatewayTimeout {
			break
		}
		if attempt >= 3 {
			break
		}
		wait := time.Duration(attempt*20) * time.Second
		if resp != nil {
			say(note, fmt.Sprintf("LoTW is busy (HTTP %d) — asking again in %s…", resp.StatusCode, wait))
			resp.Body.Close()
		} else {
			say(note, fmt.Sprintf("LoTW did not answer (%v) — asking again in %s…", err, wait))
		}
		select {
		case <-ctx.Done():
			return "", ctx.Err()
		case <-time.After(wait):
		}
		req = req.Clone(ctx)
	}
	if err != nil {
		return "", fmt.Errorf("lotw: request failed: %w", err)
	}
	defer resp.Body.Close()
	stopBeat()
	say(note, fmt.Sprintf("LoTW answered (HTTP %d) — receiving…", resp.StatusCode))
	body, err := readWithProgress(ctx, resp.Body, note)
	if err != nil {
		return "", fmt.Errorf("lotw: read response: %w", err)
	}
	text := string(body)
	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("lotw: http %d", resp.StatusCode)
	}
	// Not ADIF. Two very different failures land here, and telling them apart is
	// the difference between a fixable message and a wall of markup.
	if !strings.Contains(strings.ToUpper(text), "<EOH>") && !strings.Contains(strings.ToLower(text), "<eor>") {
		trimmed := strings.TrimSpace(text)
		// Keep the whole thing in the log — that is where a real diagnosis happens,
		// and a 200-character excerpt of an HTML page tells nobody anything.
		snippet := trimmed
		if len(snippet) > 2000 {
			snippet = snippet[:2000]
		}
		LogSink("lotw: expected ADIF, got %d bytes of non-ADIF; first 2000: %s", len(text), snippet)

		// LoTW answers a REJECTED LOGIN with its ordinary web page rather than an
		// error string, so an HTML body here means the credentials were not
		// accepted — not that the download is broken.
		lower := strings.ToLower(trimmed)
		if strings.HasPrefix(lower, "<!doctype html") || strings.HasPrefix(lower, "<html") || strings.Contains(lower, "logbook of the world</title>") {
			return "", fmt.Errorf("LoTW returned its web page instead of a log, which is how it answers a login it did not accept. " +
				"Check the username and password in Settings → External services: LoTW wants your lotw.arrl.org WEBSITE login, " +
				"not your callsign certificate or your ARRL member number")
		}

		// Anything else: a plain-text complaint from LoTW, or a maintenance notice.
		msg := trimmed
		if len(msg) > 200 {
			msg = msg[:200] + "…"
		}
		return "", fmt.Errorf("lotw: unexpected response: %s", msg)
	}
	return text, nil
}

// LoTW uploads go through TQSL (ARRL's Trusted QSL signer): there is no
// plain HTTP API — every QSO must be signed with the station certificate
// before LoTW accepts it. We write the QSO to a temporary ADIF file and run
// tqsl in batch mode to sign and upload it in one shot.

// StationLocation is one TQSL "Station Location" the user has defined. These
// pair a callsign with a certificate + grid/zones; the upload picks one by
// name (the -l flag).
type StationLocation struct {
	Name string `json:"name"`
	Call string `json:"call"`
	Grid string `json:"grid"`
	DXCC int    `json:"dxcc"`
}

// stationDataFile mirrors TQSL's station_data XML.
type stationDataFile struct {
	XMLName  xml.Name `xml:"StationDataFile"`
	Stations []struct {
		Name string `xml:"name,attr"`
		Call string `xml:"CALL"`
		Grid string `xml:"GRIDSQUARE"`
		DXCC int    `xml:"DXCC"`
	} `xml:"StationData"`
}

// ListStationLocations parses TQSL's station_data file and returns the
// defined locations. Used to populate the Station Location dropdown — the
// same file Log4OM reads.
func ListStationLocations(stationDataPath string) ([]StationLocation, error) {
	data, err := os.ReadFile(stationDataPath)
	if err != nil {
		return nil, fmt.Errorf("read station_data: %w", err)
	}
	var f stationDataFile
	if err := xml.Unmarshal(data, &f); err != nil {
		return nil, fmt.Errorf("parse station_data: %w", err)
	}
	out := make([]StationLocation, 0, len(f.Stations))
	for _, s := range f.Stations {
		out = append(out, StationLocation{Name: s.Name, Call: s.Call, Grid: s.Grid, DXCC: s.DXCC})
	}
	return out, nil
}

func fileExists(p string) bool {
	info, err := os.Stat(p)
	return err == nil && !info.IsDir()
}

// UploadLoTW signs and uploads one ADIF record via TQSL. tempDir is where the
// temporary .adi is written (falls back to the OS temp dir). Returns OK when
// LoTW accepts the QSO or reports it as a duplicate (already uploaded).
//
// TQSL command:
//
//	tqsl -d -x -a all -l "<location>" -u [-p <keypass>] <file.adi>
//
// Exit codes are TQSL's own (see the table in its cmdline help). 8 and 9 are
// the ones that matter and both used to be read as plain success: 8 means NO
// QSOs were processed and 9 means some were left out — in each case because
// they were already uploaded OR outside the callsign certificate's date range.
// Reporting either as success is how a contact came to be stamped "uploaded"
// while LoTW had never seen it.
// scrubMyCnty removes MY_CNTY fields TQSL would refuse. LoTW's secondary
// subdivisions are the US county enumeration — "XX,County" with a two-letter
// state — and TQSL rejects the whole record over anything else, so a Canadian
// station's "ONTARIO,Kawartha" (or a bare county) must simply not be sent.
// MY_STATE and MY_GRIDSQUARE already locate the station for LoTW.
var myCntyRe = regexp.MustCompile(`(?i)<MY_CNTY:([0-9]+)(?::[A-Za-z])?>`)

func scrubMyCnty(adif string) string {
	for {
		loc := myCntyRe.FindStringSubmatchIndex(adif)
		if loc == nil {
			return adif
		}
		n, _ := strconv.Atoi(adif[loc[2]:loc[3]])
		end := loc[1] + n
		if end > len(adif) {
			end = len(adif)
		}
		val := adif[loc[1]:end]
		if len(val) > 3 && val[2] == ',' {
			// "XX,..." — the US shape TQSL accepts; leave it for the county hunters.
			rest := scrubMyCnty(adif[end:])
			return adif[:end] + rest
		}
		adif = adif[:loc[0]] + strings.TrimLeft(adif[end:], " ")
	}
}

func UploadLoTW(ctx context.Context, cfg ServiceConfig, tempDir, adifRecord string) (UploadResult, error) {
	adifRecord = scrubMyCnty(adifRecord)
	tqsl := strings.TrimSpace(cfg.TQSLPath)
	loc := strings.TrimSpace(cfg.StationLocation)
	switch {
	case tqsl == "":
		return UploadResult{}, fmt.Errorf("lotw: TQSL path not set")
	case !fileExists(tqsl):
		return UploadResult{}, fmt.Errorf("lotw: TQSL not found at %q", tqsl)
	case loc == "":
		return UploadResult{}, fmt.Errorf("lotw: station location not set")
	case strings.TrimSpace(adifRecord) == "":
		return UploadResult{}, fmt.Errorf("lotw: empty adif record")
	}

	// Write the QSO to a temp ADIF file (minimal header keeps strict TQSL
	// happy). Cleaned up after upload.
	if strings.TrimSpace(tempDir) == "" {
		tempDir = os.TempDir()
	}
	f, err := os.CreateTemp(tempDir, "opslog-lotw-*.adi")
	if err != nil {
		return UploadResult{}, fmt.Errorf("lotw: create temp file: %w", err)
	}
	tmpPath := f.Name()
	defer os.Remove(tmpPath)
	if _, err := f.WriteString("OpsLog LoTW upload\n<PROGRAMID:6>OpsLog <EOH>\n" + adifRecord + "\n"); err != nil {
		f.Close()
		return UploadResult{}, fmt.Errorf("lotw: write temp file: %w", err)
	}
	f.Close()

	args := []string{"-d", "-x", "-a", "all", "-l", loc, "-u"}
	if pwd := strings.TrimSpace(cfg.KeyPassword); pwd != "" {
		args = append(args, "-p", pwd)
	}
	if cfg.WriteLog {
		// -t writes a TQSL diagnostic log; drop it next to the temp ADIF.
		args = append(args, "-t", filepath.Join(tempDir, "opslog-tqsl.log"))
	}
	args = append(args, tmpPath)

	// TQSL launches a child process and contacts LoTW — give it generous
	// time, independent of any short caller deadline.
	runCtx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
	defer cancel()
	_ = ctx

	cmd := exec.CommandContext(runCtx, tqsl, args...)
	out, runErr := cmd.CombinedOutput()
	msg := strings.TrimSpace(string(out))

	code := 0
	if runErr != nil {
		if ee, ok := runErr.(*exec.ExitError); ok {
			code = ee.ExitCode()
		} else if errors.Is(runErr, syscall.Errno(740)) || strings.Contains(strings.ToLower(runErr.Error()), "requires elevation") {
			// ERROR_ELEVATION_REQUIRED (740): tqsl.exe is set to require admin
			// rights (its "Run as administrator" compatibility flag, or an
			// AppCompat RUNASADMIN entry), but OpsLog isn't elevated so Windows
			// refuses to launch it. Actionable message instead of the raw error.
			return UploadResult{}, fmt.Errorf(
				"lotw: Windows won't launch tqsl.exe because it's marked \"Run as administrator\". "+
					"Fix: right-click %q → Properties → Compatibility → UNTICK \"Run this program as an administrator\" (Apply). "+
					"Or run OpsLog itself as administrator.", tqsl)
		} else {
			return UploadResult{}, fmt.Errorf("lotw: run tqsl: %w", runErr)
		}
	}

	// TQSL's exit codes, from its own cmdline documentation. Two of them used to
	// be read as plain success, and that is how contacts came to be stamped
	// "uploaded" while LoTW had never seen them:
	//
	//	8  NO QSOs were processed — already uploaded OR OUT OF DATE RANGE
	//	9  some processed, some ignored — same two reasons
	//	14 some already uploaded, the rest signed
	//
	// "Out of date range" is the one that bites: a contact older than the
	// callsign certificate's validity is silently left out, and reporting that as
	// success stamped it sent for ever. TQSL says which case it is in its output,
	// so the message is carried up rather than replaced with a guess.
	if code != 0 && LogSink != nil {
		// The ADIF that was handed to TQSL, verbatim, and how TQSL was called.
		//
		// TQSL says "no QSOs processed" for several unrelated reasons — already
		// uploaded, outside the certificate's dates, or a STATION_CALLSIGN that
		// does not match the station location it was told to sign with — and its
		// message does not distinguish them. An operator whose contact was
		// refused, then accepted after a round trip through another logger, is
		// reporting a difference in THIS RECORD, and there is no way to see it
		// afterwards: the temp file is deleted as soon as TQSL returns.
		//
		// Nothing secret here: it is a QSO, and the key password is not logged.
		LogSink("lotw: tqsl exit %d for station location %q", code, loc)
		LogSink("lotw: record was: %s", strings.TrimSpace(adifRecord))
	}

	switch code {
	case 0:
		return UploadResult{OK: true, Message: "uploaded to LoTW"}, nil
	case 9, 14:
		return UploadResult{OK: true, Ignored: true, Message: tqslDetail(msg,
			"uploaded — but TQSL left some contacts out (already uploaded, or outside the certificate's date range)")}, nil
	case 8:
		// Nothing reached LoTW. NOT stamped as sent: a duplicate left at "R" is
		// harmless and will be refused again, while a contact wrongly marked sent
		// is one the operator will never think to look at again.
		return UploadResult{OK: false, Ignored: true, Message: tqslDetail(msg,
				"TQSL uploaded nothing — every contact was already uploaded, or outside the certificate's date range")},
			fmt.Errorf("lotw: no QSOs processed")
	default:
		if msg == "" {
			msg = fmt.Sprintf("tqsl exit code %d", code)
		}
		return UploadResult{OK: false, Message: msg}, fmt.Errorf("lotw: tqsl failed (code %d): %s", code, msg)
	}
}

// tqslDetail keeps the lines of TQSL's own output that say what happened to the
// contacts, and appends the summary.
//
// TQSL is explicit — "414 QSO records were already uploaded", "N QSO records
// are out of date range" — and that sentence is the whole answer to "why is my
// contact not on LoTW". It used to be captured and thrown away.
func tqslDetail(out, summary string) string {
	var keep []string
	for _, ln := range strings.Split(out, "\n") {
		ln = strings.TrimSpace(ln)
		l := strings.ToLower(ln)
		if strings.Contains(l, "qso") && (strings.Contains(l, "already uploaded") ||
			strings.Contains(l, "date range") || strings.Contains(l, "ignored") ||
			strings.Contains(l, "duplicate")) {
			keep = append(keep, ln)
		}
	}
	if len(keep) == 0 {
		return summary
	}
	return summary + " — " + strings.Join(keep, "; ")
}

// TestLoTW validates the LoTW config: tqsl present and the chosen station
// location exists in station_data.
func TestLoTW(cfg ServiceConfig, stationDataPath string) (string, error) {
	tqsl := strings.TrimSpace(cfg.TQSLPath)
	loc := strings.TrimSpace(cfg.StationLocation)
	if tqsl == "" || !fileExists(tqsl) {
		return "", fmt.Errorf("lotw: TQSL not found (set the TQSL path)")
	}
	if loc == "" {
		return "", fmt.Errorf("lotw: pick a station location")
	}
	locs, err := ListStationLocations(stationDataPath)
	if err != nil {
		return "", fmt.Errorf("lotw: can't read station locations: %w", err)
	}
	found := ""
	for _, l := range locs {
		if strings.EqualFold(l.Name, loc) {
			found = l.Call
			break
		}
	}
	if found == "" {
		return "", fmt.Errorf("lotw: station location %q not found in TQSL", loc)
	}

	// LoTW is TWO credentials doing two jobs, and the button used to report only
	// the first. Uploading goes through TQSL and is signed by the certificate —
	// the website password is never involved, so a wrong one breaks nothing until
	// the day confirmations are downloaded and nobody connects the two events.
	//
	// So the download login is tested separately, and said separately. A future
	// "since" date makes LoTW return an empty report rather than the whole
	// account: the credentials are what is being checked, not the log.
	up := fmt.Sprintf("Ready — TQSL found, location %q (%s)", loc, found)
	if strings.TrimSpace(cfg.Username) == "" || cfg.Password == "" {
		return up + ". Download login not set — confirmations cannot be fetched.", nil
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	if _, err := DownloadLoTWConfirmations(ctx, nil, cfg, "2099-01-01", "", false, nil); err != nil {
		return "", fmt.Errorf("%s — but the DOWNLOAD login failed: %w", up, err)
	}
	return up + ". Download login accepted.", nil
}