fix: while connected to MySQL if internet was lost no qso would be logged anymore

internal/adif/export.go

@@ -120,6 +120,18 @@ func SingleRecordADIF(q qso.QSO) string {
 	return b.String()
 }
 
+// FullRecordADIF serialises one QSO LOSSLESSLY — including the APP_* extras —
+// so it can be written out and read back with nothing dropped. Used by the
+// offline queue: a QSO parked in the safety file must come back identical
+// (extras carry its queue id, awards refs, ADIF 3.1.7 leftovers…).
+func FullRecordADIF(q qso.QSO) string {
+	var b strings.Builder
+	bw := bufio.NewWriter(&b)
+	writeRecord(bw, q, true)
+	bw.Flush()
+	return b.String()
+}
+
 // BatchRecordsADIF wraps already-serialised records (each terminated by <EOR>,
 // e.g. from SingleRecordADIF) in a minimal ADIF document with a standard
 // header. Used by file-based batch upload APIs such as Club Log's putlogs.php.

internal/db/connloss.go

@@ -0,0 +1,66 @@
+package db
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"errors"
+	"net"
+	"strings"
+
+	"github.com/go-sql-driver/mysql"
+)
+
+// IsConnLost reports whether err means the database was UNREACHABLE (network
+// down, server gone, dead pooled connection) as opposed to a legitimate
+// server-side SQL error (constraint violation, bad data, syntax).
+//
+// This distinction is the linchpin of the offline safety net: a QSO is parked in
+// the local ADIF outbox ONLY on a connection loss. If we queued on any error, a
+// genuine data bug would silently vanish into the file instead of being
+// reported — the worst possible outcome.
+//
+// A *mysql.MySQLError means the SERVER answered and rejected us, so it is never
+// a connection loss, whatever its code.
+func IsConnLost(err error) bool {
+	if err == nil {
+		return false
+	}
+	// The server responded → a real SQL error, not a lost link.
+	var me *mysql.MySQLError
+	if errors.As(err, &me) {
+		return false
+	}
+	if errors.Is(err, driver.ErrBadConn) || errors.Is(err, sql.ErrConnDone) || errors.Is(err, sql.ErrTxDone) {
+		return true
+	}
+	var ne net.Error
+	if errors.As(err, &ne) {
+		return true
+	}
+	var oe *net.OpError
+	if errors.As(err, &oe) {
+		return true
+	}
+	// The MySQL driver reports a few of these as plain strings.
+	s := strings.ToLower(err.Error())
+	for _, p := range []string{
+		"invalid connection",
+		"bad connection",
+		"connection refused",
+		"connection reset",
+		"broken pipe",
+		"no such host",
+		"i/o timeout",
+		"dial tcp",
+		"unexpected eof",
+		"driver: bad connection",
+		"can't connect",
+		"network is unreachable",
+		"host is unreachable",
+	} {
+		if strings.Contains(s, p) {
+			return true
+		}
+	}
+	return false
+}

internal/extsvc/clublog.go

@@ -21,6 +21,12 @@ const clublogRealtimeURL = "https://clublog.org/realtime.php"
 // N QSOs is one HTTP request instead of N realtime.php calls.
 const clublogBatchURL = "https://clublog.org/putlogs.php"
 
+// clublogUserAgent identifies OpsLog to Club Log. Go's default
+// "Go-http-client/1.1" User-Agent is blocked by Club Log's web front end (nginx
+// returns 403 Forbidden before the request reaches the app), so every request
+// must send a real, app-identifying User-Agent.
+const clublogUserAgent = "OpsLog/1.0 (+https://github.com/GregTroar/OpsLog)"
+
 // clublogAppAPIKey is OpsLog's Club Log *application* API key. Club Log
 // requires an api parameter that identifies the client software (not the
 // user) — the same way Log4OM embeds its own key — so we ship it baked in
@@ -122,6 +128,7 @@ func UploadClublogADIF(ctx context.Context, client *http.Client, cfg ServiceConf
 		return UploadResult{}, fmt.Errorf("clublog: build request: %w", err)
 	}
 	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.Header.Set("User-Agent", clublogUserAgent)
 	if client == nil {
 		client = &http.Client{Timeout: 120 * time.Second}
 	}
@@ -135,10 +142,63 @@ func UploadClublogADIF(ctx context.Context, client *http.Client, cfg ServiceConf
 	if resp.StatusCode == http.StatusOK {
 		return UploadResult{OK: true, Message: msg}, nil
 	}
-	if msg == "" {
-		msg = fmt.Sprintf("HTTP %d", resp.StatusCode)
+	diag := clublogFailureDiag(resp, msg)
+	return UploadResult{OK: false, Message: diag}, fmt.Errorf("clublog: batch upload failed: %s", diag)
+}
+
+// clublogFailureDiag turns a non-200 response into a readable one-liner that
+// names WHO blocked the request — Club Log's app vs. an intermediary (Cloudflare,
+// Sucuri, a corporate proxy, an antivirus TLS shim). A bare "403 Forbidden nginx"
+// page means the request never reached the app; the fingerprint headers below
+// tell the operator whether it's Club Log's own WAF or something on their side.
+func clublogFailureDiag(resp *http.Response, body string) string {
+	server := strings.TrimSpace(resp.Header.Get("Server"))
+	// Notable intermediary fingerprints — presence points at the culprit.
+	fp := []string{}
+	for _, h := range []string{"CF-RAY", "CF-Mitigated", "X-Sucuri-ID", "X-Sucuri-Block", "X-Squid-Error", "Via", "X-Cache", "Retry-After"} {
+		if v := strings.TrimSpace(resp.Header.Get(h)); v != "" {
+			fp = append(fp, h+"="+v)
+		}
 	}
-	return UploadResult{OK: false, Message: msg}, fmt.Errorf("clublog: batch upload failed: %s", msg)
+	// Collapse the HTML error page to something short.
+	summary := stripHTMLBrief(body)
+	if summary == "" {
+		summary = fmt.Sprintf("HTTP %d", resp.StatusCode)
+	}
+	out := fmt.Sprintf("HTTP %d — %s", resp.StatusCode, summary)
+	if server != "" {
+		out += " [server=" + server + "]"
+	}
+	if len(fp) > 0 {
+		out += " [" + strings.Join(fp, " ") + "]"
+	}
+	return out
+}
+
+// stripHTMLBrief removes tags and collapses whitespace, returning the first ~160
+// chars of visible text — enough to read "403 Forbidden" without the markup.
+func stripHTMLBrief(s string) string {
+	var b strings.Builder
+	depth := 0
+	for _, r := range s {
+		switch r {
+		case '<':
+			depth++
+		case '>':
+			if depth > 0 {
+				depth--
+			}
+		default:
+			if depth == 0 {
+				b.WriteRune(r)
+			}
+		}
+	}
+	out := strings.Join(strings.Fields(b.String()), " ")
+	if len(out) > 160 {
+		out = out[:160] + "…"
+	}
+	return out
 }
 
 // TestClublog validates the configured credentials by attempting a no-op
@@ -164,6 +224,7 @@ func clublogPost(ctx context.Context, client *http.Client, endpoint string, form
 		return UploadResult{}, fmt.Errorf("clublog: build request: %w", err)
 	}
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("User-Agent", clublogUserAgent)
 	if client == nil {
 		client = &http.Client{Timeout: 20 * time.Second}
 	}

internal/extsvc/lotw.go

@@ -3,6 +3,7 @@ package extsvc
 import (
 	"context"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,6 +12,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 )
 
@@ -212,6 +214,15 @@ func UploadLoTW(ctx context.Context, cfg ServiceConfig, tempDir, adifRecord stri
 	if runErr != nil {
 		if ee, ok := runErr.(*exec.ExitError); ok {
 			code = ee.ExitCode()
+		} else if errors.Is(runErr, syscall.Errno(740)) || strings.Contains(strings.ToLower(runErr.Error()), "requires elevation") {
+			// ERROR_ELEVATION_REQUIRED (740): tqsl.exe is set to require admin
+			// rights (its "Run as administrator" compatibility flag, or an
+			// AppCompat RUNASADMIN entry), but OpsLog isn't elevated so Windows
+			// refuses to launch it. Actionable message instead of the raw error.
+			return UploadResult{}, fmt.Errorf(
+				"lotw: Windows won't launch tqsl.exe because it's marked \"Run as administrator\". " +
+					"Fix: right-click %q → Properties → Compatibility → UNTICK \"Run this program as an administrator\" (Apply). " +
+					"Or run OpsLog itself as administrator.", tqsl)
 		} else {
 			return UploadResult{}, fmt.Errorf("lotw: run tqsl: %w", runErr)
 		}

internal/offlineq/offlineq.go

@@ -0,0 +1,202 @@
+// Package offlineq is OpsLog's offline safety net.
+//
+// When the shared MySQL logbook is unreachable, a QSO must never be lost just
+// because the network blinked. Instead of failing, the QSO is appended to a
+// local ADIF file (the "outbox") and replayed into the database as soon as it
+// comes back — then the file is ARCHIVED, never deleted.
+//
+// Deliberately NOT a sync engine: it only ever PUSHES the operator's own QSOs.
+// There is no mirror, no pull, no merge, no tombstones — which is exactly why it
+// stays small. The cost, accepted by design: during an outage you don't see other
+// operators' QSOs and the worked-before check doesn't know about your pending
+// ones. See the queue view in the UI for what's waiting.
+//
+// The file lives in OpsLog's data directory — NEVER in a cloud-synced folder
+// (Seafile/OneDrive): replicating a live file byte-by-byte is what we're
+// escaping in the first place.
+package offlineq
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"hamlog/internal/adif"
+	"hamlog/internal/qso"
+)
+
+// FileName is the outbox. Pending QSOs accumulate here while the DB is down.
+const FileName = "opslog-pending.adi"
+
+// Queue owns the outbox file. All operations are serialised: the logging path
+// (append) and the replay loop (read/rewrite/archive) run on different
+// goroutines.
+type Queue struct {
+	dir string
+	mu  sync.Mutex
+}
+
+// New returns a queue backed by <dir>/opslog-pending.adi.
+func New(dir string) *Queue { return &Queue{dir: strings.TrimSpace(dir)} }
+
+// Path is the outbox file's location (shown in the UI so the operator always
+// knows where their QSOs physically are).
+func (q *Queue) Path() string { return filepath.Join(q.dir, FileName) }
+
+// newQueueID mints the id that makes replay idempotent.
+func newQueueID() string {
+	var b [12]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return fmt.Sprintf("t%d", time.Now().UnixNano())
+	}
+	return hex.EncodeToString(b[:])
+}
+
+// Append parks a QSO in the outbox, stamping it with a queue id (returned) so a
+// repeated replay can recognise it. The file is created with an ADIF header on
+// first use, then appended to — an append can't corrupt what's already there.
+func (q *Queue) Append(rec qso.QSO) (string, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	if rec.Extras == nil {
+		rec.Extras = map[string]string{}
+	}
+	qid := strings.TrimSpace(rec.Extras[qso.OfflineQueueKey])
+	if qid == "" {
+		qid = newQueueID()
+		rec.Extras[qso.OfflineQueueKey] = qid
+	}
+
+	if err := os.MkdirAll(q.dir, 0o755); err != nil {
+		return "", fmt.Errorf("offlineq: create dir: %w", err)
+	}
+	path := q.Path()
+	_, statErr := os.Stat(path)
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return "", fmt.Errorf("offlineq: open %s: %w", path, err)
+	}
+	defer f.Close()
+
+	var b strings.Builder
+	if os.IsNotExist(statErr) { // brand-new file → write the ADIF header once
+		b.WriteString("OpsLog offline queue — QSOs logged while the database was unreachable.\n")
+		b.WriteString("<ADIF_VER:5>3.1.4 <PROGRAMID:6>OpsLog <EOH>\n\n")
+	}
+	b.WriteString(strings.TrimRight(adif.FullRecordADIF(rec), "\r\n"))
+	b.WriteString("\n")
+
+	if _, err := f.WriteString(b.String()); err != nil {
+		return "", fmt.Errorf("offlineq: write: %w", err)
+	}
+	// Flush to disk: the whole point is surviving a crash/power cut.
+	if err := f.Sync(); err != nil {
+		return "", fmt.Errorf("offlineq: sync: %w", err)
+	}
+	return qid, nil
+}
+
+// Pending parses the outbox into QSOs (each carrying its queue id in Extras).
+// A missing file simply means nothing is pending.
+func (q *Queue) Pending() ([]qso.QSO, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.pendingLocked()
+}
+
+func (q *Queue) pendingLocked() ([]qso.QSO, error) {
+	f, err := os.Open(q.Path())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	var out []qso.QSO
+	err = adif.Parse(f, func(rec adif.Record) error {
+		if v, ok := adif.RecordToQSO(rec); ok {
+			out = append(out, v)
+		}
+		return nil
+	})
+	if err != nil {
+		return out, fmt.Errorf("offlineq: parse %s: %w", q.Path(), err)
+	}
+	return out, nil
+}
+
+// Count is how many QSOs are waiting (0 when the file is absent).
+func (q *Queue) Count() int {
+	p, err := q.Pending()
+	if err != nil {
+		return 0
+	}
+	return len(p)
+}
+
+// Rewrite replaces the outbox with exactly these QSOs — used after a replay to
+// keep only the ones that FAILED. Written to a temp file and renamed, so a crash
+// mid-write can't truncate the queue.
+func (q *Queue) Rewrite(keep []qso.QSO) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.rewriteLocked(keep)
+}
+
+func (q *Queue) rewriteLocked(keep []qso.QSO) error {
+	path := q.Path()
+	if len(keep) == 0 {
+		// Nothing left: remove the (now empty) outbox. The caller archives the
+		// original content first, so this never destroys the only copy.
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("offlineq: remove: %w", err)
+		}
+		return nil
+	}
+	var b strings.Builder
+	b.WriteString("OpsLog offline queue — QSOs logged while the database was unreachable.\n")
+	b.WriteString("<ADIF_VER:5>3.1.4 <PROGRAMID:6>OpsLog <EOH>\n\n")
+	for _, v := range keep {
+		b.WriteString(strings.TrimRight(adif.FullRecordADIF(v), "\r\n"))
+		b.WriteString("\n")
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(b.String()), 0o644); err != nil {
+		return fmt.Errorf("offlineq: write temp: %w", err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		return fmt.Errorf("offlineq: replace: %w", err)
+	}
+	return nil
+}
+
+// Archive copies the outbox to a timestamped file BEFORE it is cleared, so the
+// QSOs always exist somewhere on disk even if the replay later turns out to have
+// gone wrong. Deleting the only copy of someone's contacts is the one mistake
+// you don't get to undo.
+func (q *Queue) Archive() (string, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	data, err := os.ReadFile(q.Path())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", err
+	}
+	name := fmt.Sprintf("opslog-pending-%s.adi", time.Now().Format("2006-01-02-1504"))
+	dst := filepath.Join(q.dir, name)
+	if err := os.WriteFile(dst, data, 0o644); err != nil {
+		return "", fmt.Errorf("offlineq: archive: %w", err)
+	}
+	return dst, nil
+}

internal/offlineq/offlineq_test.go

@@ -0,0 +1,93 @@
+package offlineq
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"hamlog/internal/qso"
+)
+
+// The outbox is the ONLY copy of a QSO logged while the database was down, so the
+// round-trip must be lossless: append → read back identical (callsign, band, mode,
+// date) and each record must carry a queue id (what makes the replay idempotent).
+func TestQueueRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	q := New(dir)
+
+	if n := q.Count(); n != 0 {
+		t.Fatalf("fresh queue: count = %d, want 0", n)
+	}
+
+	mk := func(call, band, mode string) qso.QSO {
+		return qso.QSO{
+			Callsign: call, Band: band, Mode: mode,
+			QSODate: time.Date(2026, 7, 10, 12, 34, 0, 0, time.UTC),
+		}
+	}
+	id1, err := q.Append(mk("K1ABC", "20m", "SSB"))
+	if err != nil {
+		t.Fatalf("append 1: %v", err)
+	}
+	id2, err := q.Append(mk("DL1XYZ", "40m", "CW"))
+	if err != nil {
+		t.Fatalf("append 2: %v", err)
+	}
+	if id1 == "" || id2 == "" || id1 == id2 {
+		t.Fatalf("queue ids must be non-empty and distinct: %q / %q", id1, id2)
+	}
+
+	pending, err := q.Pending()
+	if err != nil {
+		t.Fatalf("pending: %v", err)
+	}
+	if len(pending) != 2 {
+		t.Fatalf("pending = %d, want 2", len(pending))
+	}
+	if pending[0].Callsign != "K1ABC" || pending[0].Band != "20m" || pending[0].Mode != "SSB" {
+		t.Errorf("record 1 round-tripped wrong: %+v", pending[0])
+	}
+	if pending[1].Callsign != "DL1XYZ" || pending[1].Mode != "CW" {
+		t.Errorf("record 2 round-tripped wrong: %+v", pending[1])
+	}
+	// The queue id must survive the ADIF round-trip — without it the replay
+	// can't be idempotent and a crash would duplicate contacts.
+	for i, p := range pending {
+		if p.Extras[qso.OfflineQueueKey] == "" {
+			t.Errorf("record %d lost its %s extra", i, qso.OfflineQueueKey)
+		}
+	}
+
+	// Archive keeps a copy BEFORE we clear anything.
+	arch, err := q.Archive()
+	if err != nil || arch == "" {
+		t.Fatalf("archive: %v (path %q)", err, arch)
+	}
+	if _, err := os.Stat(arch); err != nil {
+		t.Fatalf("archive file missing: %v", err)
+	}
+
+	// Partial replay: the first QSO went in, the second failed → keep only it.
+	if err := q.Rewrite(pending[1:]); err != nil {
+		t.Fatalf("rewrite: %v", err)
+	}
+	left, err := q.Pending()
+	if err != nil {
+		t.Fatalf("pending after rewrite: %v", err)
+	}
+	if len(left) != 1 || left[0].Callsign != "DL1XYZ" {
+		t.Fatalf("after rewrite: got %d records (%v), want just DL1XYZ", len(left), left)
+	}
+
+	// Everything replayed → the outbox is removed (the archive still holds it).
+	if err := q.Rewrite(nil); err != nil {
+		t.Fatalf("rewrite empty: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, FileName)); !os.IsNotExist(err) {
+		t.Errorf("outbox should be gone once fully replayed, stat err = %v", err)
+	}
+	if n := q.Count(); n != 0 {
+		t.Errorf("count after full replay = %d, want 0", n)
+	}
+}

internal/qso/qso.go

@@ -1011,6 +1011,32 @@ func FilterableFields() []string {
 }
 
 // columnExpr resolves a filter field to a safe SQL expression — either a
+// OfflineQueueKey is the ADIF extras key stamped on a QSO that was parked in the
+// offline safety file. It survives the ADIF round-trip and makes the replay
+// IDEMPOTENT: if the app dies between "inserted into the DB" and "removed from
+// the file", the next replay sees the id already in the log and skips it instead
+// of creating a duplicate.
+const OfflineQueueKey = "APP_OPSLOG_QUEUEID"
+
+// ExistsByQueueID reports whether a QSO carrying this offline-queue id is already
+// in the logbook — the guard that makes replaying the safety file safe to repeat.
+func (r *Repo) ExistsByQueueID(ctx context.Context, qid string) (bool, error) {
+	qid = strings.TrimSpace(qid)
+	if qid == "" {
+		return false, nil
+	}
+	expr := "json_extract(extras_json, '$." + OfflineQueueKey + "')"
+	if db.IsMySQL() {
+		expr = "JSON_UNQUOTE(JSON_EXTRACT(NULLIF(extras_json,''), '$." + OfflineQueueKey + "'))"
+	}
+	var n int
+	if err := r.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM qso WHERE `+expr+` = ?`, qid).Scan(&n); err != nil {
+		return false, err
+	}
+	return n > 0, nil
+}
+
 // whitelisted column name or a json_extract over extras_json.
 func columnExpr(field string) (string, bool) {
 	f := strings.ToLower(strings.TrimSpace(field))