refactor(sync): no mass backfill — identities are stamped on what is touched
Correcting an over-design of my own. I had contacts given an identity in bulk the first time synchronisation was switched on, and a full dedupe-key map of the logbook rebuilt on every pass, both sized for 123 000 contacts. Neither is needed, because the change log starts EMPTY. Contacts logged before synchronisation was switched on are never in anyone's log and so are never exchanged; an identity is stamped only on a contact that is actually logged, edited or deleted from then on. Seeding a second machine with the existing history is a one-time copy of the database or an ADIF import — not something synchronisation should be doing, and not something it can do from a file that starts empty. One case survives, and it is why the dedupe key stays. Two machines can already hold the SAME old contact, the second seeded by that copy or import, with different row ids and no identity on either. The day one of them edits it, it stamps an identity and sends a change naming it; the other has never seen that identity and would insert a duplicate. Matching on the contact itself — callsign, minute, band, mode, the importer's own key — recognises it. So that became a targeted lookup through idx_qso_callsign, run only when an identity is unknown, instead of a whole-table map rebuilt each pass. Rare work, priced as rare work.
This commit is contained in:
@@ -14,7 +14,11 @@
|
||||
-- index a TEXT column without a prefix length, and a migration that tries dies
|
||||
-- with error 1170 on every startup thereafter, with no way out from the UI.
|
||||
--
|
||||
-- Empty on every existing row: they are given an identity the first time
|
||||
-- synchronisation is switched on, in one bulk statement, not row by row.
|
||||
-- Empty on every existing row, and it STAYS empty on most of them. The change
|
||||
-- log starts empty too, so contacts logged before synchronisation was switched
|
||||
-- on are never exchanged: nothing has to be copied across, and an identity is
|
||||
-- stamped only on a contact that is actually logged, edited or deleted from
|
||||
-- then on. Seeding a second machine with the existing log is a one-time copy of
|
||||
-- the database or an ADIF import, not something synchronisation does.
|
||||
ALTER TABLE qso ADD COLUMN sync_uid TEXT NOT NULL DEFAULT '';
|
||||
CREATE INDEX IF NOT EXISTS idx_qso_sync_uid ON qso (sync_uid);
|
||||
|
||||
+30
-94
@@ -3449,103 +3449,39 @@ func (r *Repo) IDBySyncUID(ctx context.Context, uid string) (int64, bool, error)
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// CountWithoutSyncUID reports how many contacts still have no identity — what
|
||||
// the settings panel shows before the first synchronisation.
|
||||
func (r *Repo) CountWithoutSyncUID(ctx context.Context) (int64, error) {
|
||||
var n int64
|
||||
err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM qso WHERE sync_uid = '' OR sync_uid IS NULL`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NeedSyncUID returns the ids of contacts still without an identity, at most
|
||||
// limit of them.
|
||||
// IDByDedupeKey finds the local row a change refers to when its identity is
|
||||
// unknown here.
|
||||
//
|
||||
// Batched rather than "all of them": a 120 000-QSO logbook is given its
|
||||
// identities a few thousand at a time, so the first synchronisation cannot hold
|
||||
// the database — or the interface — for however long a single enormous
|
||||
// transaction would take on a remote MySQL.
|
||||
func (r *Repo) NeedSyncUID(ctx context.Context, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
limit = 2000
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id FROM qso WHERE sync_uid = '' OR sync_uid IS NULL ORDER BY id LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// AssignSyncUIDs stamps a batch of identities in ONE transaction.
|
||||
// The change log starts EMPTY, so contacts logged before synchronisation was
|
||||
// switched on are never exchanged — nothing has to be copied across, and
|
||||
// identities are stamped lazily on the contacts that are actually touched.
|
||||
//
|
||||
// One statement per row inside a single transaction, not one transaction per
|
||||
// row: on a remote MySQL the round trip dominates, and 120 000 separate commits
|
||||
// is the difference between a minute and an afternoon.
|
||||
func (r *Repo) AssignSyncUIDs(ctx context.Context, uids map[int64]string) error {
|
||||
if len(uids) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
stmt, err := tx.PrepareContext(ctx, `UPDATE qso SET sync_uid = ? WHERE id = ? AND (sync_uid = '' OR sync_uid IS NULL)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for id, uid := range uids {
|
||||
if _, err := stmt.ExecContext(ctx, uid, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SyncUIDByDedupeKey maps every contact's dedupe key to its sync identity.
|
||||
// One case still needs this. Two machines can already hold the SAME old contact
|
||||
// — the second was seeded by copying the database or importing an ADIF — with
|
||||
// different row ids and no identity on either. The day one of them edits that
|
||||
// contact, it stamps an identity and sends a change naming it; the other has
|
||||
// never seen that identity and would insert a duplicate. Matching on the
|
||||
// contact itself (callsign + minute + band + mode, the importer's own dedupe
|
||||
// key) recognises it instead.
|
||||
//
|
||||
// This is what stops two machines that already hold the SAME imported log from
|
||||
// copying it to each other: an incoming contact whose identity is unknown but
|
||||
// whose key matches one here is the same contact, and it adopts the incoming
|
||||
// identity instead of being inserted a second time.
|
||||
//
|
||||
// One scan of four short columns, done once per synchronisation pass — the same
|
||||
// shape as the importer's own dedupe load, which is the cheapest read in the
|
||||
// repo.
|
||||
func (r *Repo) SyncUIDByDedupeKey(ctx context.Context) (map[string]struct {
|
||||
ID int64
|
||||
UID string
|
||||
}, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, sync_uid, callsign, substr(qso_date, 1, 16), band, mode FROM qso`)
|
||||
// Looked up only when an identity is unknown, which is rare, and it goes
|
||||
// through idx_qso_callsign rather than scanning — a full map of every contact,
|
||||
// rebuilt each pass, was the wrong shape for something this occasional.
|
||||
func (r *Repo) IDByDedupeKey(ctx context.Context, callsign, qsoDateMinute, band, mode string) (id int64, uid string, found bool, err error) {
|
||||
if callsign == "" || qsoDateMinute == "" {
|
||||
return 0, "", false, nil
|
||||
}
|
||||
var u sql.NullString
|
||||
err = r.db.QueryRowContext(ctx, `
|
||||
SELECT id, sync_uid FROM qso
|
||||
WHERE callsign = ? AND substr(qso_date, 1, 16) = ? AND band = ? AND mode = ?
|
||||
LIMIT 1`,
|
||||
strings.ToUpper(callsign), qsoDateMinute, band, mode).Scan(&id, &u)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return 0, "", false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]struct {
|
||||
ID int64
|
||||
UID string
|
||||
}, 1024)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var uid, call, when, band, mode sql.NullString
|
||||
if err := rows.Scan(&id, &uid, &call, &when, &band, &mode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[DedupeKey(call.String, when.String, band.String, mode.String)] = struct {
|
||||
ID int64
|
||||
UID string
|
||||
}{ID: id, UID: uid.String}
|
||||
}
|
||||
return out, rows.Err()
|
||||
return id, u.String, true, nil
|
||||
}
|
||||
|
||||
@@ -108,63 +108,38 @@ func TestIDBySyncUID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Backfill: batched, and it must never overwrite an identity already there.
|
||||
func TestAssignSyncUIDsOnlyFillsTheEmptyOnes(t *testing.T) {
|
||||
r := openRepo(t)
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 8, 16, 14, 0, 0, 0, time.UTC)
|
||||
a := addQSO(t, r, "M0AAA", base)
|
||||
b := addQSO(t, r, "M0BBB", base.Add(time.Minute))
|
||||
_ = r.SetSyncUID(ctx, a, "already")
|
||||
|
||||
n, err := r.CountWithoutSyncUID(ctx)
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("CountWithoutSyncUID = %d (%v), want 1", n, err)
|
||||
}
|
||||
ids, err := r.NeedSyncUID(ctx, 100)
|
||||
if err != nil || len(ids) != 1 || ids[0] != b {
|
||||
t.Fatalf("NeedSyncUID = %v (%v), want [%d]", ids, err, b)
|
||||
}
|
||||
|
||||
// Try to overwrite BOTH; only the empty one may take.
|
||||
if err := r.AssignSyncUIDs(ctx, map[int64]string{a: "stomp", b: "fresh"}); err != nil {
|
||||
t.Fatalf("AssignSyncUIDs: %v", err)
|
||||
}
|
||||
qa, _ := r.GetByID(ctx, a)
|
||||
qb, _ := r.GetByID(ctx, b)
|
||||
if qa.SyncUID != "already" {
|
||||
t.Errorf("an existing identity was overwritten: %q", qa.SyncUID)
|
||||
}
|
||||
if qb.SyncUID != "fresh" {
|
||||
t.Errorf("the empty one was not filled: %q", qb.SyncUID)
|
||||
}
|
||||
if n, _ := r.CountWithoutSyncUID(ctx); n != 0 {
|
||||
t.Errorf("%d contacts still without an identity", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Two machines holding the SAME imported log must not copy it to each other.
|
||||
// An incoming contact with an unknown identity but a matching dedupe key is the
|
||||
// same contact, and adopts the incoming id rather than being inserted twice.
|
||||
func TestSyncUIDByDedupeKeyFindsTheSameContact(t *testing.T) {
|
||||
// Two machines can already hold the SAME old contact — the second was seeded by
|
||||
// copying the database or importing an ADIF — with different row ids and no
|
||||
// identity on either. The day one of them edits it, it stamps an identity and
|
||||
// sends a change naming it; the other has never seen that identity and would
|
||||
// insert a duplicate. Matching on the contact itself recognises it instead.
|
||||
func TestIDByDedupeKeyRecognisesTheSameContact(t *testing.T) {
|
||||
r := openRepo(t)
|
||||
ctx := context.Background()
|
||||
when := time.Date(2026, 8, 16, 14, 32, 0, 0, time.UTC)
|
||||
id := addQSO(t, r, "M0ABC", when)
|
||||
|
||||
byKey, err := r.SyncUIDByDedupeKey(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("SyncUIDByDedupeKey: %v", err)
|
||||
minute := when.UTC().Format("2006-01-02T15:04")
|
||||
gotID, uid, found, err := r.IDByDedupeKey(ctx, "M0ABC", minute, "20m", "CW")
|
||||
if err != nil || !found {
|
||||
t.Fatalf("IDByDedupeKey = (%d,%q,%v,%v), want it found", gotID, uid, found, err)
|
||||
}
|
||||
key := DedupeKey("M0ABC", when.UTC().Format("2006-01-02T15:04"), "20m", "CW")
|
||||
hit, ok := byKey[key]
|
||||
if !ok {
|
||||
t.Fatalf("key %q not found among %d entries", key, len(byKey))
|
||||
if gotID != id {
|
||||
t.Errorf("resolved to id %d, want %d", gotID, id)
|
||||
}
|
||||
if hit.ID != id {
|
||||
t.Errorf("key resolved to id %d, want %d", hit.ID, id)
|
||||
if uid != "" {
|
||||
t.Errorf("uid = %q, want empty until something stamps it", uid)
|
||||
}
|
||||
if hit.UID != "" {
|
||||
t.Errorf("UID = %q, want empty before any stamping", hit.UID)
|
||||
|
||||
// A contact this machine does not have must NOT match something else.
|
||||
if _, _, found, _ := r.IDByDedupeKey(ctx, "M0ABC", minute, "40m", "CW"); found {
|
||||
t.Error("a different band matched — the key must be all four parts")
|
||||
}
|
||||
if _, _, found, _ := r.IDByDedupeKey(ctx, "M0XYZ", minute, "20m", "CW"); found {
|
||||
t.Error("a different callsign matched")
|
||||
}
|
||||
// An empty key must never match anything.
|
||||
if _, _, found, _ := r.IDByDedupeKey(ctx, "", "", "", ""); found {
|
||||
t.Error("an empty key matched a row")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user