441 lines
13 KiB
Go
441 lines
13 KiB
Go
package adif
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"hamlog/internal/qso"
|
|
)
|
|
|
|
// ImportResult summarises an ADIF import for the UI.
|
|
type ImportResult struct {
|
|
Total int `json:"total"` // records found in the file
|
|
Imported int `json:"imported"` // successfully inserted
|
|
Skipped int `json:"skipped"` // dropped (missing required fields)
|
|
Duplicates int `json:"duplicates"` // matched an existing or earlier-in-file QSO
|
|
DuplicateSamples []string `json:"duplicate_samples"` // up to maxDuplicateSamples human-readable IDs of skipped duplicates
|
|
Errors []string `json:"errors"` // up to maxErrors error messages
|
|
}
|
|
|
|
const (
|
|
maxErrors = 50
|
|
maxDuplicateSamples = 200 // cap memory + JSON payload size
|
|
)
|
|
|
|
// Importer streams an ADI file into a QSO repository.
|
|
type Importer struct {
|
|
Repo *qso.Repo
|
|
BatchSize int // 0 → 500
|
|
SkipDuplicates bool // when true, records matching an existing or earlier-in-file QSO are skipped; otherwise all are inserted
|
|
}
|
|
|
|
// ImportFile opens the file at path and imports it into the repo.
|
|
func (im *Importer) ImportFile(ctx context.Context, path string) (ImportResult, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return ImportResult{}, fmt.Errorf("open %s: %w", path, err)
|
|
}
|
|
defer f.Close()
|
|
return im.Import(ctx, f)
|
|
}
|
|
|
|
// Import streams the ADI content from r into the repo.
|
|
func (im *Importer) Import(ctx context.Context, r interface {
|
|
Read(p []byte) (int, error)
|
|
}) (ImportResult, error) {
|
|
if im.BatchSize <= 0 {
|
|
im.BatchSize = 500
|
|
}
|
|
res := ImportResult{}
|
|
batch := make([]qso.QSO, 0, im.BatchSize)
|
|
|
|
// One upfront query for every existing dedup key — cheaper than N
|
|
// per-record EXISTS calls. The same map gets new keys appended as we
|
|
// import so duplicates inside the file are caught too. Loaded
|
|
// unconditionally so the "duplicates" count is meaningful even when
|
|
// SkipDuplicates is off (the user still wants to know how many).
|
|
seen, err := im.Repo.ExistingDedupeKeys(ctx)
|
|
if err != nil {
|
|
return res, fmt.Errorf("load dedupe keys: %w", err)
|
|
}
|
|
|
|
flush := func() error {
|
|
if len(batch) == 0 {
|
|
return nil
|
|
}
|
|
n, err := im.Repo.AddBatch(ctx, batch)
|
|
res.Imported += int(n)
|
|
batch = batch[:0]
|
|
return err
|
|
}
|
|
|
|
err = Parse(r, func(rec Record) error {
|
|
res.Total++
|
|
q, ok := recordToQSO(rec)
|
|
if !ok {
|
|
res.Skipped++
|
|
if len(res.Errors) < maxErrors {
|
|
res.Errors = append(res.Errors,
|
|
fmt.Sprintf("record %d: missing required fields (call/band/mode/date)", res.Total))
|
|
}
|
|
return nil
|
|
}
|
|
key := qso.DedupeKey(q.Callsign, q.QSODate.UTC().Format("2006-01-02T15:04"), q.Band, q.Mode)
|
|
if _, dup := seen[key]; dup {
|
|
res.Duplicates++
|
|
if len(res.DuplicateSamples) < maxDuplicateSamples {
|
|
res.DuplicateSamples = append(res.DuplicateSamples,
|
|
fmt.Sprintf("%s · %s · %s · %s",
|
|
q.Callsign,
|
|
q.QSODate.UTC().Format("2006-01-02 15:04"),
|
|
q.Band, q.Mode))
|
|
}
|
|
if im.SkipDuplicates {
|
|
return nil
|
|
}
|
|
// Fall through: insert anyway. Don't add to seen[] — keeps
|
|
// the duplicate count meaningful if the same row appears 3x.
|
|
} else {
|
|
seen[key] = struct{}{}
|
|
}
|
|
batch = append(batch, q)
|
|
if len(batch) >= im.BatchSize {
|
|
return flush()
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
_ = flush()
|
|
return res, err
|
|
}
|
|
if err := flush(); err != nil {
|
|
return res, err
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// adifPromoted lists every lowercase ADIF tag that maps to a promoted column.
|
|
// Anything not in this set ends up in Extras.
|
|
var adifPromoted = stringSet(
|
|
// Core
|
|
"call", "qso_date", "time_on", "qso_date_off", "time_off",
|
|
"band", "band_rx", "mode", "submode", "freq", "freq_rx",
|
|
"rst_sent", "rst_rcvd",
|
|
// Contacted
|
|
"name", "qth", "address", "email", "web",
|
|
"gridsquare", "gridsquare_ext", "vucc_grids",
|
|
"country", "state", "cnty",
|
|
"dxcc", "cont", "cqz", "ituz",
|
|
"iota", "sota_ref", "pota_ref",
|
|
"age", "lat", "lon", "rig", "ant",
|
|
// QSL
|
|
"qsl_sent", "qsl_rcvd",
|
|
"qslsdate", "qslrdate", "qsl_via", "qslmsg", "qslmsg_rcvd",
|
|
"lotw_qsl_sent", "lotw_qsl_rcvd", "lotw_qslsdate", "lotw_qslrdate",
|
|
"eqsl_qsl_sent", "eqsl_qsl_rcvd", "eqsl_qslsdate", "eqsl_qslrdate",
|
|
"clublog_qso_upload_date", "clublog_qso_upload_status",
|
|
"hrdlog_qso_upload_date", "hrdlog_qso_upload_status",
|
|
// Contest
|
|
"contest_id", "srx", "stx", "srx_string", "stx_string",
|
|
"check", "precedence", "arrl_sect",
|
|
// Sat / propagation
|
|
"prop_mode", "sat_name", "sat_mode", "ant_az", "ant_el", "ant_path",
|
|
// My station
|
|
"station_callsign", "operator",
|
|
"my_gridsquare", "my_gridsquare_ext", "my_country", "my_state", "my_cnty", "my_iota",
|
|
"my_sota_ref", "my_pota_ref",
|
|
"my_dxcc", "my_cq_zone", "my_itu_zone", "my_lat", "my_lon",
|
|
"my_street", "my_city", "my_postal_code", "my_rig", "my_antenna",
|
|
// Misc
|
|
"tx_pwr", "comment", "notes",
|
|
)
|
|
|
|
func stringSet(items ...string) map[string]struct{} {
|
|
m := make(map[string]struct{}, len(items))
|
|
for _, s := range items {
|
|
m[s] = struct{}{}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// recordToQSO maps an ADIF record onto a QSO. Returns false if required
|
|
// fields are missing. Any ADIF tag we don't promote is stored in Extras.
|
|
func recordToQSO(rec Record) (qso.QSO, bool) {
|
|
call := strings.ToUpper(strings.TrimSpace(rec["call"]))
|
|
if call == "" {
|
|
return qso.QSO{}, false
|
|
}
|
|
band := strings.ToLower(strings.TrimSpace(rec["band"]))
|
|
mode := strings.ToUpper(strings.TrimSpace(rec["mode"]))
|
|
submode := strings.ToUpper(strings.TrimSpace(rec["submode"]))
|
|
date := parseDateTime(rec["qso_date"], rec["time_on"])
|
|
if date.IsZero() || band == "" || mode == "" {
|
|
return qso.QSO{}, false
|
|
}
|
|
// ADIF promotes specific digital flavours into the SUBMODE field with
|
|
// a generic parent in MODE (e.g. MODE=MFSK SUBMODE=FT4). Loggers
|
|
// uniformly display the submode in that case — promote it so all
|
|
// downstream code (entry strip, worked-before, exports) sees "FT4"
|
|
// rather than "MFSK". SSB+USB/LSB and RTTY+JOH stay on the parent
|
|
// because the parent IS the displayed mode there.
|
|
if submode != "" && submodeSubsumesParent(submode) {
|
|
mode = submode
|
|
submode = "" // redundant once promoted
|
|
}
|
|
|
|
q := qso.QSO{
|
|
Callsign: call,
|
|
QSODate: date,
|
|
QSODateOff: parseDateTime(rec["qso_date_off"], rec["time_off"]),
|
|
Band: band,
|
|
BandRX: strings.ToLower(rec["band_rx"]),
|
|
Mode: mode,
|
|
Submode: submode,
|
|
}
|
|
if hz, ok := parseFreqHz(rec["freq"]); ok {
|
|
q.FreqHz = &hz
|
|
}
|
|
if hz, ok := parseFreqHz(rec["freq_rx"]); ok {
|
|
q.FreqRXHz = &hz
|
|
}
|
|
|
|
q.RSTSent = rec["rst_sent"]
|
|
q.RSTRcvd = rec["rst_rcvd"]
|
|
|
|
// Contacted station
|
|
q.Name = rec["name"]
|
|
q.QTH = rec["qth"]
|
|
q.Address = rec["address"]
|
|
q.Email = rec["email"]
|
|
q.Web = rec["web"]
|
|
q.Grid = strings.ToUpper(rec["gridsquare"])
|
|
q.GridExt = strings.ToUpper(rec["gridsquare_ext"])
|
|
q.VUCCGrids = strings.ToUpper(rec["vucc_grids"])
|
|
q.Country = rec["country"]
|
|
q.State = strings.ToUpper(rec["state"])
|
|
q.County = rec["cnty"]
|
|
if v, ok := parseInt(rec["dxcc"]); ok {
|
|
q.DXCC = &v
|
|
}
|
|
q.Continent = strings.ToUpper(rec["cont"])
|
|
if v, ok := parseInt(rec["cqz"]); ok {
|
|
q.CQZ = &v
|
|
}
|
|
if v, ok := parseInt(rec["ituz"]); ok {
|
|
q.ITUZ = &v
|
|
}
|
|
q.IOTA = strings.ToUpper(rec["iota"])
|
|
q.SOTARef = strings.ToUpper(rec["sota_ref"])
|
|
q.POTARef = strings.ToUpper(rec["pota_ref"])
|
|
if v, ok := parseInt(rec["age"]); ok {
|
|
q.Age = &v
|
|
}
|
|
if v, ok := parseFloat(rec["lat"]); ok {
|
|
q.Lat = &v
|
|
}
|
|
if v, ok := parseFloat(rec["lon"]); ok {
|
|
q.Lon = &v
|
|
}
|
|
q.Rig = rec["rig"]
|
|
q.Ant = rec["ant"]
|
|
|
|
// QSL
|
|
q.QSLSent = rec["qsl_sent"]
|
|
q.QSLRcvd = rec["qsl_rcvd"]
|
|
q.QSLSentDate = rec["qslsdate"]
|
|
q.QSLRcvdDate = rec["qslrdate"]
|
|
q.QSLVia = rec["qsl_via"]
|
|
q.QSLMsg = rec["qslmsg"]
|
|
q.QSLMsgRcvd = rec["qslmsg_rcvd"]
|
|
q.LOTWSent = rec["lotw_qsl_sent"]
|
|
q.LOTWRcvd = rec["lotw_qsl_rcvd"]
|
|
q.LOTWSentDate = rec["lotw_qslsdate"]
|
|
q.LOTWRcvdDate = rec["lotw_qslrdate"]
|
|
q.EQSLSent = rec["eqsl_qsl_sent"]
|
|
q.EQSLRcvd = rec["eqsl_qsl_rcvd"]
|
|
q.EQSLSentDate = rec["eqsl_qslsdate"]
|
|
q.EQSLRcvdDate = rec["eqsl_qslrdate"]
|
|
q.ClublogUploadDate = rec["clublog_qso_upload_date"]
|
|
q.ClublogUploadStatus = rec["clublog_qso_upload_status"]
|
|
q.HRDLogUploadDate = rec["hrdlog_qso_upload_date"]
|
|
q.HRDLogUploadStatus = rec["hrdlog_qso_upload_status"]
|
|
|
|
// Contest
|
|
q.ContestID = rec["contest_id"]
|
|
if v, ok := parseInt(rec["srx"]); ok {
|
|
q.SRX = &v
|
|
}
|
|
if v, ok := parseInt(rec["stx"]); ok {
|
|
q.STX = &v
|
|
}
|
|
q.SRXString = rec["srx_string"]
|
|
q.STXString = rec["stx_string"]
|
|
q.Check = rec["check"]
|
|
q.Precedence = rec["precedence"]
|
|
q.ARRLSect = strings.ToUpper(rec["arrl_sect"])
|
|
|
|
// Sat / propagation
|
|
q.PropMode = strings.ToUpper(rec["prop_mode"])
|
|
q.SatName = strings.ToUpper(rec["sat_name"])
|
|
q.SatMode = rec["sat_mode"]
|
|
if v, ok := parseFloat(rec["ant_az"]); ok {
|
|
q.AntAz = &v
|
|
}
|
|
if v, ok := parseFloat(rec["ant_el"]); ok {
|
|
q.AntEl = &v
|
|
}
|
|
q.AntPath = strings.ToUpper(rec["ant_path"])
|
|
|
|
// My station
|
|
q.StationCallsign = strings.ToUpper(rec["station_callsign"])
|
|
q.Operator = strings.ToUpper(rec["operator"])
|
|
q.MyGrid = strings.ToUpper(rec["my_gridsquare"])
|
|
q.MyGridExt = strings.ToUpper(rec["my_gridsquare_ext"])
|
|
q.MyCountry = rec["my_country"]
|
|
q.MyState = strings.ToUpper(rec["my_state"])
|
|
q.MyCounty = rec["my_cnty"]
|
|
q.MyIOTA = strings.ToUpper(rec["my_iota"])
|
|
q.MySOTARef = strings.ToUpper(rec["my_sota_ref"])
|
|
q.MyPOTARef = strings.ToUpper(rec["my_pota_ref"])
|
|
if v, ok := parseInt(rec["my_dxcc"]); ok {
|
|
q.MyDXCC = &v
|
|
}
|
|
if v, ok := parseInt(rec["my_cq_zone"]); ok {
|
|
q.MyCQZone = &v
|
|
}
|
|
if v, ok := parseInt(rec["my_itu_zone"]); ok {
|
|
q.MyITUZone = &v
|
|
}
|
|
if v, ok := parseFloat(rec["my_lat"]); ok {
|
|
q.MyLat = &v
|
|
}
|
|
if v, ok := parseFloat(rec["my_lon"]); ok {
|
|
q.MyLon = &v
|
|
}
|
|
q.MyStreet = rec["my_street"]
|
|
q.MyCity = rec["my_city"]
|
|
q.MyPostalCode = rec["my_postal_code"]
|
|
q.MyRig = rec["my_rig"]
|
|
q.MyAntenna = rec["my_antenna"]
|
|
|
|
// Misc
|
|
if v, ok := parseFloat(rec["tx_pwr"]); ok {
|
|
q.TXPower = &v
|
|
}
|
|
q.Comment = rec["comment"]
|
|
q.Notes = rec["notes"]
|
|
|
|
// Everything else lands in extras (uppercased ADIF names).
|
|
var extras map[string]string
|
|
for k, v := range rec {
|
|
if _, ok := adifPromoted[k]; ok {
|
|
continue
|
|
}
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if extras == nil {
|
|
extras = map[string]string{}
|
|
}
|
|
extras[strings.ToUpper(k)] = v
|
|
}
|
|
q.Extras = extras
|
|
|
|
return q, true
|
|
}
|
|
|
|
// parseDateTime combines ADIF QSO_DATE (YYYYMMDD) with TIME (HHMMSS or HHMM).
|
|
func parseDateTime(date, timeStr string) time.Time {
|
|
date = strings.TrimSpace(date)
|
|
timeStr = strings.TrimSpace(timeStr)
|
|
if len(date) != 8 {
|
|
return time.Time{}
|
|
}
|
|
layout := "20060102"
|
|
val := date
|
|
if len(timeStr) == 4 {
|
|
layout = "200601021504"
|
|
val = date + timeStr
|
|
} else if len(timeStr) == 6 {
|
|
layout = "20060102150405"
|
|
val = date + timeStr
|
|
}
|
|
t, err := time.ParseInLocation(layout, val, time.UTC)
|
|
if err != nil {
|
|
return time.Time{}
|
|
}
|
|
return t.UTC()
|
|
}
|
|
|
|
func parseFreqHz(s string) (int64, bool) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
mhz, err := strconv.ParseFloat(s, 64)
|
|
if err != nil || mhz <= 0 {
|
|
return 0, false
|
|
}
|
|
return int64(mhz*1_000_000 + 0.5), true
|
|
}
|
|
|
|
func parseInt(s string) (int, bool) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
v, err := strconv.Atoi(s)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return v, true
|
|
}
|
|
|
|
func parseFloat(s string) (float64, bool) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
v, err := strconv.ParseFloat(s, 64)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return v, true
|
|
}
|
|
|
|
// promotableSubmodes are SUBMODE values that uniquely identify the mode
|
|
// every logger displays — the ADIF parent (MFSK, DATA, PSK, JT…) is
|
|
// generic and unhelpful to show. SSB's USB/LSB and RTTY's JOH are NOT
|
|
// here because operators want to see "SSB" / "RTTY", not the sideband
|
|
// or RTTY variant.
|
|
var promotableSubmodes = map[string]bool{
|
|
// MFSK family (modern FT/Q65/JS8/MSK144 live here)
|
|
"FT2": true, "FT4": true, "FT8": true, "JS8": true, "MSK144": true, "ISCAT": true,
|
|
"Q65": true, "FST4": true, "FST4W": true,
|
|
"MFSK16": true, "MFSK32": true, "MFSK64": true, "MFSK128": true,
|
|
"OLIVIA": true,
|
|
// JT family (some loggers still parent these under DATA)
|
|
"JT65": true, "JT9": true, "JT4": true, "JT6M": true, "JT44": true, "T10": true,
|
|
// PSK family
|
|
"PSK31": true, "PSK63": true, "PSK125": true, "PSK250": true, "PSK500": true,
|
|
"QPSK31": true, "QPSK63": true, "QPSK125": true, "QPSK250": true, "QPSK500": true,
|
|
// THOR / DOMINO / HELL
|
|
"THOR4": true, "THOR8": true, "THOR16": true, "THOR32": true,
|
|
"DOMINOF": true, "DOMINOEX": true,
|
|
"HELL80": true, "FMHELL": true,
|
|
// DIGITALVOICE variants
|
|
"FREEDV": true,
|
|
// VARA family — Log4OM parents these under DYNAMIC. VarAC chat uses
|
|
// "VARA HF" (with the space) so we match both spellings.
|
|
"VARA": true, "VARA HF": true, "VARA FM": true, "VARAC": true,
|
|
}
|
|
|
|
func submodeSubsumesParent(submode string) bool {
|
|
return promotableSubmodes[submode]
|
|
}
|