From 45b9bcdea7048400262d4b296611a49504277099 Mon Sep 17 00:00:00 2001 From: rouggy Date: Sun, 5 Jul 2026 10:58:49 +0200 Subject: [PATCH] feat: New tool to remove duplicates in the log --- app.go | 78 ++++++++++ frontend/src/App.tsx | 9 ++ frontend/src/components/DuplicatesModal.tsx | 162 ++++++++++++++++++++ frontend/src/lib/i18n.tsx | 15 +- frontend/wailsjs/go/main/App.d.ts | 2 + frontend/wailsjs/go/main/App.js | 4 + frontend/wailsjs/go/models.ts | 32 ++++ 7 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 frontend/src/components/DuplicatesModal.tsx diff --git a/app.go b/app.go index 8e5a638..064cd18 100644 --- a/app.go +++ b/app.go @@ -3368,6 +3368,84 @@ func (a *App) DeleteQSOs(ids []int64) (int64, error) { return a.qso.DeleteMany(a.ctx, ids) } +// DuplicateGroup is a set of QSOs the log considers the same contact. +type DuplicateGroup struct { + Key string `json:"key"` + QSOs []qso.QSO `json:"qsos"` +} + +// normDupeMode collapses SSB sidebands so USB/LSB count as the same slot; other +// modes (FT8/RTTY/CW…) stay distinct so they aren't wrongly grouped. +func normDupeMode(m string) string { + u := strings.ToUpper(strings.TrimSpace(m)) + if u == "USB" || u == "LSB" { + return "SSB" + } + return u +} + +// FindDuplicates scans the whole log and returns groups of duplicate QSOs: same +// callsign + band + mode (USB/LSB folded to SSB) logged within windowMinutes of +// each other — the real signature of an accidental double-log, which lands a few +// minutes apart, not necessarily the same minute. QSOs of a call+band+mode are +// sorted by time and split into clusters wherever the gap to the next exceeds +// the window, so two genuinely separate contacts hours apart are NOT grouped. +// windowMinutes <= 0 falls back to same UTC day. Each group's QSOs are +// oldest-first and groups are ordered by date, so the UI can pre-select all but +// the first. +func (a *App) FindDuplicates(windowMinutes int) []DuplicateGroup { + if a.qso == nil { + return nil + } + byKey := map[string][]qso.QSO{} + _ = a.qso.IterateAll(a.ctx, func(q qso.QSO) error { + call := strings.ToUpper(strings.TrimSpace(q.Callsign)) + if call == "" { + return nil + } + key := call + "|" + strings.ToLower(strings.TrimSpace(q.Band)) + "|" + normDupeMode(q.Mode) + byKey[key] = append(byKey[key], q) + return nil + }) + + out := []DuplicateGroup{} + for key, qs := range byKey { + if len(qs) < 2 { + continue + } + sort.Slice(qs, func(i, j int) bool { return qs[i].QSODate.Before(qs[j].QSODate) }) + if windowMinutes <= 0 { + // Whole-day fallback: split on a change of UTC calendar day. + i := 0 + for i < len(qs) { + j := i + 1 + for j < len(qs) && qs[j].QSODate.UTC().Format("2006-01-02") == qs[i].QSODate.UTC().Format("2006-01-02") { + j++ + } + if j-i > 1 { + out = append(out, DuplicateGroup{Key: fmt.Sprintf("%s|%d", key, i), QSOs: append([]qso.QSO(nil), qs[i:j]...)}) + } + i = j + } + continue + } + win := time.Duration(windowMinutes) * time.Minute + i := 0 + for i < len(qs) { + j := i + 1 + for j < len(qs) && qs[j].QSODate.Sub(qs[j-1].QSODate) <= win { + j++ + } + if j-i > 1 { + out = append(out, DuplicateGroup{Key: fmt.Sprintf("%s|%d", key, i), QSOs: append([]qso.QSO(nil), qs[i:j]...)}) + } + i = j + } + } + sort.Slice(out, func(i, j int) bool { return out[i].QSOs[0].QSODate.Before(out[j].QSOs[0].QSODate) }) + return out +} + // QSLBulkUpdate carries the paper-QSL fields to apply to a selection. An empty // string leaves that field unchanged (so you can set only "received = Y + date" // without touching the sent side). diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index d314893..a8f7a14 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -70,6 +70,7 @@ import { cleanSpotter, inferSpotMode, spotModeCategory, spotStatusKey } from '@/ import { WorkedBeforeGrid } from '@/components/WorkedBeforeGrid'; import { NetControlPanel } from '@/components/NetControlPanel'; import { ContestPanel, CONTEST_DEFAULT, type ContestSession } from '@/components/ContestPanel'; +import { DuplicatesModal } from '@/components/DuplicatesModal'; import { useI18n } from '@/lib/i18n'; import { AlertsModal } from '@/components/AlertsModal'; import { BulkEditModal } from '@/components/BulkEditModal'; @@ -907,6 +908,7 @@ export default function App() { const [settingsSection, setSettingsSection] = useState(undefined); const [showDeleteAll, setShowDeleteAll] = useState(false); const [showAbout, setShowAbout] = useState(false); + const [showDuplicates, setShowDuplicates] = useState(false); const [updateInfo, setUpdateInfo] = useState<{ latest: string; url: string } | null>(null); // Check GitHub for a newer release once at startup (unless disabled in // General); surface a toast if one exists. Best effort — silent on failure. @@ -2325,6 +2327,8 @@ export default function App() { { type: 'item', label: (netEnabled ? '✓ ' : '') + t('tools.net'), action: 'tools.net' }, { type: 'item', label: t('tools.alerts'), action: 'tools.alerts' }, { type: 'separator' }, + { type: 'item', label: t('tools.duplicates'), action: 'tools.duplicates', disabled: total === 0 }, + { type: 'separator' }, // Maintenance — bumped here while we only have one entry. Will move // to a Tools → Maintenance submenu once Clublog + LoTW refresh land. { type: 'item', label: ctyRefreshing ? 'Refreshing cty.dat…' : 'Refresh cty.dat', action: 'tools.refreshCty', disabled: ctyRefreshing }, @@ -2354,6 +2358,7 @@ export default function App() { case 'tools.cwdecoder': toggleCwDecoder(); break; case 'tools.net': setNetEnabled((v) => { const nv = !v; if (nv) setActiveTab('net'); return nv; }); break; case 'tools.alerts': setAlertsOpen(true); break; + case 'tools.duplicates': setShowDuplicates(true); break; case 'tools.refreshCty': refreshCtyDat(); break; case 'tools.downloadRefs': downloadRefs(); break; case 'help.about': setShowAbout(true); break; @@ -4251,6 +4256,10 @@ export default function App() { setAlertsOpen(false)} bands={bands} modes={modes} countries={countries} /> )} + {showDuplicates && ( + setShowDuplicates(false)} onDeleted={() => refresh()} /> + )} + showToast(`OpsLog QSL sent to ${call}`)} onError={(msg) => showToast(msg)} diff --git a/frontend/src/components/DuplicatesModal.tsx b/frontend/src/components/DuplicatesModal.tsx new file mode 100644 index 0000000..176aca9 --- /dev/null +++ b/frontend/src/components/DuplicatesModal.tsx @@ -0,0 +1,162 @@ +import { useEffect, useState } from 'react'; +import { Copy, Loader2, Trash2, X } from 'lucide-react'; +import { FindDuplicates, DeleteQSOs } from '../../wailsjs/go/main/App'; +import { useI18n } from '@/lib/i18n'; +import { cn } from '@/lib/utils'; + +type QSO = { + id: number; callsign: string; qso_date: string; band?: string; mode?: string; + freq_hz?: number; rst_sent?: string; rst_rcvd?: string; name?: string; +}; +type Group = { key: string; qsos: QSO[] }; + +const fmtDate = (iso: string) => { + const d = new Date(iso); + if (isNaN(d.getTime())) return iso; + const p = (n: number) => String(n).padStart(2, '0'); + return `${d.getUTCFullYear()}-${p(d.getUTCMonth() + 1)}-${p(d.getUTCDate())} ${p(d.getUTCHours())}:${p(d.getUTCMinutes())}`; +}; +const fmtFreq = (hz?: number) => (hz && hz > 0 ? (hz / 1e6).toFixed(3) : '—'); + +// DuplicatesModal — scans the log for duplicate QSOs (same call+band+mode on the +// same day; optionally same minute), groups them, and lets the operator pick +// which copies to delete. The oldest in each group is left unchecked by default. +export function DuplicatesModal({ onClose, onDeleted }: { onClose: () => void; onDeleted: () => void }) { + const { t } = useI18n(); + const [windowMin, setWindowMin] = useState(5); // duplicates land a few minutes apart + const [loading, setLoading] = useState(true); + const [groups, setGroups] = useState([]); + const [sel, setSel] = useState>(new Set()); + const [busy, setBusy] = useState(false); + + const load = (w: number) => { + setLoading(true); + FindDuplicates(w) + .then((g) => { + const gs = (g ?? []) as Group[]; + setGroups(gs); + // Default: select every QSO except the first (oldest) of each group. + const next = new Set(); + gs.forEach((grp) => grp.qsos.slice(1).forEach((q) => next.add(q.id))); + setSel(next); + }) + .catch(() => { setGroups([]); setSel(new Set()); }) + .finally(() => setLoading(false)); + }; + useEffect(() => { load(windowMin); /* eslint-disable-next-line */ }, [windowMin]); + + const toggle = (id: number) => setSel((s) => { const n = new Set(s); n.has(id) ? n.delete(id) : n.add(id); return n; }); + const selectExtras = () => { const n = new Set(); groups.forEach((g) => g.qsos.slice(1).forEach((q) => n.add(q.id))); setSel(n); }; + const deselectAll = () => setSel(new Set()); + + const totalQsos = groups.reduce((a, g) => a + g.qsos.length, 0); + + const doDelete = async () => { + if (sel.size === 0) return; + if (!confirm(t('dup.deleteConfirm', { n: sel.size }))) return; + setBusy(true); + try { + await DeleteQSOs(Array.from(sel) as any); + onDeleted(); + load(windowMin); // rescan so the list reflects what's left + } catch { /* ignore */ } finally { setBusy(false); } + }; + + return ( +
+
e.stopPropagation()}> + {/* Header */} +
+ + {t('dup.title')} + +
+ + {/* Controls */} +
+

{t('dup.hint')}

+
+ +
+ + +
+
+
+ {t('dup.summary', { groups: groups.length, qsos: totalQsos, sel: sel.size })} +
+
+ + {/* Body */} +
+ {loading ? ( +
+ {t('dup.scanning')} +
+ ) : groups.length === 0 ? ( +
{t('dup.none')}
+ ) : ( + groups.map((g) => ( +
+
+ {g.qsos[0].callsign} + + {g.qsos[0].band} · {g.qsos[0].mode} · {g.qsos.length}× + +
+ + + + + + + + + + + + {g.qsos.map((q, i) => { + const checked = sel.has(q.id); + return ( + + + + + + + + ); + })} + +
{t('dup.colDate')}{t('dup.colFreq')}{t('dup.colRst')}{t('dup.colName')}
+ toggle(q.id)} /> + + {fmtDate(q.qso_date)} + {i === 0 && {t('dup.keep')}} + {fmtFreq(q.freq_hz)}{q.rst_sent || '—'}/{q.rst_rcvd || '—'}{q.name || ''}
+
+ )) + )} +
+ + {/* Footer */} +
+ + +
+
+
+ ); +} diff --git a/frontend/src/lib/i18n.tsx b/frontend/src/lib/i18n.tsx index fa1d44c..8931632 100644 --- a/frontend/src/lib/i18n.tsx +++ b/frontend/src/lib/i18n.tsx @@ -21,7 +21,13 @@ const en: Dict = { 'tools.qslManager': 'QSL Manager…', 'tools.qslDesigner': 'QSL Card Designer…', 'tools.winkeyer': 'WinKeyer CW keyer', 'tools.dvk': 'Digital Voice Keyer', 'tools.cwDecoder': 'CW decoder (RX audio)', 'tools.net': 'NET Control', 'tools.alerts': 'Alert management…', - 'menu.help': 'Help', 'help.about': 'About OpsLog', + 'menu.help': 'Help', 'help.about': 'About OpsLog', 'tools.duplicates': 'Find duplicates…', + // Duplicates modal + 'dup.title': 'Find duplicates', 'dup.scanning': 'Scanning the log…', 'dup.none': 'No duplicates found. 🎉', + 'dup.hint': 'Each group is the same contact logged more than once (callsign + band + mode). The first (oldest) is left unchecked; tick the ones to delete.', + 'dup.window': 'Within', 'dup.minutes': 'min', 'dup.sameDay': 'same day', 'dup.summary': '{groups} groups · {qsos} duplicate QSOs · {sel} to delete', + 'dup.selectExtras': 'Select all but the first', 'dup.deselectAll': 'Deselect all', 'dup.deleteSel': 'Delete {n} selected', 'dup.deleteConfirm': 'Delete {n} QSO(s)? This cannot be undone.', 'dup.deleted': '{n} duplicate(s) deleted', 'dup.keep': 'oldest', 'dup.close': 'Close', + 'dup.colDate': 'Date / time (UTC)', 'dup.colFreq': 'Freq', 'dup.colRst': 'RST s/r', 'dup.colName': 'Name', 'profileScope.saved': 'Saved for profile', 'profileScope.switch': '— switch profiles to edit another identity.', // Tabs 'tab.main': 'Main', 'tab.recent': 'Recent QSOs', 'tab.cluster': 'Cluster', 'tab.worked': 'Worked before', @@ -210,7 +216,12 @@ const fr: Dict = { 'tools.qslManager': 'Gestionnaire QSL…', 'tools.qslDesigner': 'Créateur de carte QSL…', 'tools.winkeyer': 'Manipulateur CW WinKeyer', 'tools.dvk': 'Manipulateur vocal numérique', 'tools.cwDecoder': 'Décodeur CW (audio RX)', 'tools.net': 'Contrôle de NET', 'tools.alerts': 'Gestion des alertes…', - 'menu.help': 'Aide', 'help.about': 'À propos d\'OpsLog', + 'menu.help': 'Aide', 'help.about': 'À propos d\'OpsLog', 'tools.duplicates': 'Trouver les doublons…', + 'dup.title': 'Trouver les doublons', 'dup.scanning': 'Analyse du journal…', 'dup.none': 'Aucun doublon trouvé. 🎉', + 'dup.hint': 'Chaque groupe est le même contact enregistré plusieurs fois (indicatif + bande + mode). Le premier (le plus ancien) est laissé décoché ; coche ceux à supprimer.', + 'dup.window': 'En moins de', 'dup.minutes': 'min', 'dup.sameDay': 'même jour', 'dup.summary': '{groups} groupes · {qsos} QSO en double · {sel} à supprimer', + 'dup.selectExtras': 'Tout sélectionner sauf le premier', 'dup.deselectAll': 'Tout désélectionner', 'dup.deleteSel': 'Supprimer {n} sélectionné(s)', 'dup.deleteConfirm': 'Supprimer {n} QSO ? Action irréversible.', 'dup.deleted': '{n} doublon(s) supprimé(s)', 'dup.keep': 'le plus ancien', 'dup.close': 'Fermer', + 'dup.colDate': 'Date / heure (UTC)', 'dup.colFreq': 'Fréq', 'dup.colRst': 'RST e/r', 'dup.colName': 'Nom', 'profileScope.saved': 'Enregistré pour le profil', 'profileScope.switch': "— change de profil pour éditer une autre identité.", 'tab.main': 'Principal', 'tab.recent': 'QSO récents', 'tab.cluster': 'Cluster', 'tab.worked': 'Déjà contacté', 'tab.awards': 'Diplômes', 'tab.bandmap': 'Carte de bande', 'tab.contest': 'Contest', 'tab.net': 'NET', diff --git a/frontend/wailsjs/go/main/App.d.ts b/frontend/wailsjs/go/main/App.d.ts index 80b0c07..2c25e5c 100644 --- a/frontend/wailsjs/go/main/App.d.ts +++ b/frontend/wailsjs/go/main/App.d.ts @@ -144,6 +144,8 @@ export function ExportCabrilloSelected(arg1:string,arg2:Array):Promise>; +export function FindDuplicates(arg1:number):Promise>; + export function FindQSOsForUpload(arg1:string,arg2:string):Promise>; export function FlexATUBypass():Promise; diff --git a/frontend/wailsjs/go/main/App.js b/frontend/wailsjs/go/main/App.js index 7252676..4fb764e 100644 --- a/frontend/wailsjs/go/main/App.js +++ b/frontend/wailsjs/go/main/App.js @@ -250,6 +250,10 @@ export function FilterFields() { return window['go']['main']['App']['FilterFields'](); } +export function FindDuplicates(arg1) { + return window['go']['main']['App']['FindDuplicates'](arg1); +} + export function FindQSOsForUpload(arg1, arg2) { return window['go']['main']['App']['FindQSOsForUpload'](arg1, arg2); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index b84838b..7787b54 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -1528,6 +1528,38 @@ export namespace main { this.is_custom = source["is_custom"]; } } + export class DuplicateGroup { + key: string; + qsos: qso.QSO[]; + + static createFrom(source: any = {}) { + return new DuplicateGroup(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.key = source["key"]; + this.qsos = this.convertValues(source["qsos"], qso.QSO); + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } export class EmailSettings { enabled: boolean; smtp_host: string;