refactor(sync): no mass backfill — identities are stamped on what is touched
Correcting an over-design of my own. I had contacts given an identity in bulk the first time synchronisation was switched on, and a full dedupe-key map of the logbook rebuilt on every pass, both sized for 123 000 contacts. Neither is needed, because the change log starts EMPTY. Contacts logged before synchronisation was switched on are never in anyone's log and so are never exchanged; an identity is stamped only on a contact that is actually logged, edited or deleted from then on. Seeding a second machine with the existing history is a one-time copy of the database or an ADIF import — not something synchronisation should be doing, and not something it can do from a file that starts empty. One case survives, and it is why the dedupe key stays. Two machines can already hold the SAME old contact, the second seeded by that copy or import, with different row ids and no identity on either. The day one of them edits it, it stamps an identity and sends a change naming it; the other has never seen that identity and would insert a duplicate. Matching on the contact itself — callsign, minute, band, mode, the importer's own key — recognises it. So that became a targeted lookup through idx_qso_callsign, run only when an identity is unknown, instead of a whole-table map rebuilt each pass. Rare work, priced as rare work.
This commit is contained in:
+30
-94
@@ -3449,103 +3449,39 @@ func (r *Repo) IDBySyncUID(ctx context.Context, uid string) (int64, bool, error)
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
// CountWithoutSyncUID reports how many contacts still have no identity — what
|
||||
// the settings panel shows before the first synchronisation.
|
||||
func (r *Repo) CountWithoutSyncUID(ctx context.Context) (int64, error) {
|
||||
var n int64
|
||||
err := r.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM qso WHERE sync_uid = '' OR sync_uid IS NULL`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// NeedSyncUID returns the ids of contacts still without an identity, at most
|
||||
// limit of them.
|
||||
// IDByDedupeKey finds the local row a change refers to when its identity is
|
||||
// unknown here.
|
||||
//
|
||||
// Batched rather than "all of them": a 120 000-QSO logbook is given its
|
||||
// identities a few thousand at a time, so the first synchronisation cannot hold
|
||||
// the database — or the interface — for however long a single enormous
|
||||
// transaction would take on a remote MySQL.
|
||||
func (r *Repo) NeedSyncUID(ctx context.Context, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
limit = 2000
|
||||
}
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
`SELECT id FROM qso WHERE sync_uid = '' OR sync_uid IS NULL ORDER BY id LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// AssignSyncUIDs stamps a batch of identities in ONE transaction.
|
||||
// The change log starts EMPTY, so contacts logged before synchronisation was
|
||||
// switched on are never exchanged — nothing has to be copied across, and
|
||||
// identities are stamped lazily on the contacts that are actually touched.
|
||||
//
|
||||
// One statement per row inside a single transaction, not one transaction per
|
||||
// row: on a remote MySQL the round trip dominates, and 120 000 separate commits
|
||||
// is the difference between a minute and an afternoon.
|
||||
func (r *Repo) AssignSyncUIDs(ctx context.Context, uids map[int64]string) error {
|
||||
if len(uids) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
stmt, err := tx.PrepareContext(ctx, `UPDATE qso SET sync_uid = ? WHERE id = ? AND (sync_uid = '' OR sync_uid IS NULL)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for id, uid := range uids {
|
||||
if _, err := stmt.ExecContext(ctx, uid, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// SyncUIDByDedupeKey maps every contact's dedupe key to its sync identity.
|
||||
// One case still needs this. Two machines can already hold the SAME old contact
|
||||
// — the second was seeded by copying the database or importing an ADIF — with
|
||||
// different row ids and no identity on either. The day one of them edits that
|
||||
// contact, it stamps an identity and sends a change naming it; the other has
|
||||
// never seen that identity and would insert a duplicate. Matching on the
|
||||
// contact itself (callsign + minute + band + mode, the importer's own dedupe
|
||||
// key) recognises it instead.
|
||||
//
|
||||
// This is what stops two machines that already hold the SAME imported log from
|
||||
// copying it to each other: an incoming contact whose identity is unknown but
|
||||
// whose key matches one here is the same contact, and it adopts the incoming
|
||||
// identity instead of being inserted a second time.
|
||||
//
|
||||
// One scan of four short columns, done once per synchronisation pass — the same
|
||||
// shape as the importer's own dedupe load, which is the cheapest read in the
|
||||
// repo.
|
||||
func (r *Repo) SyncUIDByDedupeKey(ctx context.Context) (map[string]struct {
|
||||
ID int64
|
||||
UID string
|
||||
}, error) {
|
||||
rows, err := r.db.QueryContext(ctx, `
|
||||
SELECT id, sync_uid, callsign, substr(qso_date, 1, 16), band, mode FROM qso`)
|
||||
// Looked up only when an identity is unknown, which is rare, and it goes
|
||||
// through idx_qso_callsign rather than scanning — a full map of every contact,
|
||||
// rebuilt each pass, was the wrong shape for something this occasional.
|
||||
func (r *Repo) IDByDedupeKey(ctx context.Context, callsign, qsoDateMinute, band, mode string) (id int64, uid string, found bool, err error) {
|
||||
if callsign == "" || qsoDateMinute == "" {
|
||||
return 0, "", false, nil
|
||||
}
|
||||
var u sql.NullString
|
||||
err = r.db.QueryRowContext(ctx, `
|
||||
SELECT id, sync_uid FROM qso
|
||||
WHERE callsign = ? AND substr(qso_date, 1, 16) = ? AND band = ? AND mode = ?
|
||||
LIMIT 1`,
|
||||
strings.ToUpper(callsign), qsoDateMinute, band, mode).Scan(&id, &u)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return 0, "", false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]struct {
|
||||
ID int64
|
||||
UID string
|
||||
}, 1024)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var uid, call, when, band, mode sql.NullString
|
||||
if err := rows.Scan(&id, &uid, &call, &when, &band, &mode); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[DedupeKey(call.String, when.String, band.String, mode.String)] = struct {
|
||||
ID int64
|
||||
UID string
|
||||
}{ID: id, UID: uid.String}
|
||||
}
|
||||
return out, rows.Err()
|
||||
return id, u.String, true, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user