From 37298afd7726b10517af7ac1f2c69924dab73a94 Mon Sep 17 00:00:00 2001 From: rouggy Date: Sun, 30 Aug 2026 13:13:22 +0200 Subject: [PATCH] =?UTF-8?q?fix(qso):=20every=20by-ids=20statement=20is=20c?= =?UTF-8?q?hunked=20=E2=80=94=20168k=20QSOs=20is=20a=20selection=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One placeholder per id in a single IN (…) hits SQLite's bound-variable cap: bulk field set died at 168 000 QSOs with 'too many SQL variables' (10 000 passed). One chunker now serves bulk set (text/int/extra/frequency), the delete, the post-upload markers and the export-selection iterator — the last collecting and sorting once at the end so its chronological contract holds across chunks. 500 ids a statement keeps every backend far from any limit. --- changelog.json | 10 +++ internal/qso/qso.go | 198 ++++++++++++++++++++++++++------------------ 2 files changed, 129 insertions(+), 79 deletions(-) diff --git a/changelog.json b/changelog.json index 47a41ee..c5973ef 100644 --- a/changelog.json +++ b/changelog.json @@ -1,4 +1,14 @@ [ + { + "version": "0.27.2", + "date": "", + "en": [ + "Bulk operations work on any size of selection — setting a field, fixing frequencies, deleting, marking uploads and exporting the selection all failed with “too many SQL variables” past a few tens of thousands of QSOs. Statements are now issued in slices." + ], + "fr": [ + "Les opérations groupées fonctionnent quelle que soit la taille de la sélection — définir un champ, corriger des fréquences, supprimer, marquer les uploads et exporter la sélection échouaient avec « too many SQL variables » au-delà de quelques dizaines de milliers de QSO. Les requêtes sont désormais émises par tranches." + ] + }, { "version": "0.27.1", "date": "", diff --git a/internal/qso/qso.go b/internal/qso/qso.go index 833be8d..a65ba6a 100644 --- a/internal/qso/qso.go +++ b/internal/qso/qso.go @@ -722,15 +722,14 @@ func (r *Repo) MarkUploadedBatch(ctx context.Context, statusCol, dateCol, date s if len(ids) == 0 { return nil } - ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") - args := make([]any, 0, len(ids)+2) - args = append(args, date, db.NowISO()) - for _, id := range ids { - args = append(args, id) - } - _, err := r.db.ExecContext(ctx, - `UPDATE qso SET `+statusCol+` = 'Y', `+dateCol+` = ?, updated_at = ? WHERE id IN (`+ph+`)`, - args...) + now := db.NowISO() + _, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + args := append([]any{date, now}, idArgs...) + _, err := r.db.ExecContext(ctx, + `UPDATE qso SET `+statusCol+` = 'Y', `+dateCol+` = ?, updated_at = ? WHERE id IN (`+ph+`)`, + args...) + return 0, err + }) if err != nil { return fmt.Errorf("mark uploaded batch (%d): %w", len(ids), err) } @@ -872,6 +871,35 @@ var bulkEditableCols = map[string]bool{ // own path, not the text one: the columns are nullable integers, and while // SQLite would coerce "14" quietly, a shared MySQL logbook would not — and an // empty string is NULL here, never "". +// bulkByIDChunks runs one UPDATE per slice of ids, small enough for SQLite's +// bound-variable cap: the single IN (…) with one placeholder per id worked at +// 10 000 QSOs and failed at 168 000 with "too many SQL variables". Each call +// gets the placeholder string and the id arguments for its slice; affected +// rows are summed. 500 per statement keeps every backend far from any limit +// while costing a few hundred statements on the largest logs. +func bulkByIDChunks(ctx context.Context, ids []int64, run func(ph string, idArgs []any) (int64, error)) (int64, error) { + const chunk = 500 + var total int64 + for start := 0; start < len(ids); start += chunk { + end := start + chunk + if end > len(ids) { + end = len(ids) + } + part := ids[start:end] + ph := strings.Repeat("?,", len(part)-1) + "?" + args := make([]any, len(part)) + for i, id := range part { + args[i] = id + } + n, err := run(ph, args) + if err != nil { + return total, err + } + total += n + } + return total, nil +} + var bulkEditableIntCols = map[string]bool{ "my_dxcc": true, "my_cq_zone": true, @@ -886,23 +914,23 @@ func (r *Repo) BulkSetIntField(ctx context.Context, ids []int64, column string, if len(ids) == 0 { return 0, nil } - ph := make([]string, len(ids)) - args := make([]any, 0, len(ids)+2) var val any if v != nil { val = *v } - args = append(args, val, db.NowISO()) - for i, id := range ids { - ph[i] = "?" - args = append(args, id) - } - res, err := r.db.ExecContext(ctx, - "UPDATE qso SET "+column+" = ?, updated_at = ? WHERE id IN ("+strings.Join(ph, ",")+")", args...) + now := db.NowISO() + n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + args := append([]any{val, now}, idArgs...) + res, err := r.db.ExecContext(ctx, + "UPDATE qso SET "+column+" = ?, updated_at = ? WHERE id IN ("+ph+")", args...) + if err != nil { + return 0, err + } + return res.RowsAffected() + }) if err != nil { - return 0, fmt.Errorf("bulk set %s: %w", column, err) + return n, fmt.Errorf("bulk set %s: %w", column, err) } - n, _ := res.RowsAffected() return n, nil } @@ -913,13 +941,6 @@ func (r *Repo) BulkSetField(ctx context.Context, ids []int64, column, value stri if len(ids) == 0 { return 0, nil } - ph := make([]string, len(ids)) - args := make([]any, 0, len(ids)+2) - args = append(args, value, db.NowISO()) - for i, id := range ids { - ph[i] = "?" - args = append(args, id) - } set := column + " = ?, updated_at = ?" if column == "mode" { // A submode belongs to the mode it was recorded under. Left behind, it @@ -928,13 +949,19 @@ func (r *Repo) BulkSetField(ctx context.Context, ids []int64, column, value stri // only outcome that leaves the row meaning what the operator asked for. set += ", submode = ''" } - res, err := r.db.ExecContext(ctx, - `UPDATE qso SET `+set+` WHERE id IN (`+strings.Join(ph, ",")+`)`, - args...) + now := db.NowISO() + n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + args := append([]any{value, now}, idArgs...) + res, err := r.db.ExecContext(ctx, + `UPDATE qso SET `+set+` WHERE id IN (`+ph+`)`, args...) + if err != nil { + return 0, err + } + return res.RowsAffected() + }) if err != nil { - return 0, fmt.Errorf("bulk set %s: %w", column, err) + return n, fmt.Errorf("bulk set %s: %w", column, err) } - n, _ := res.RowsAffected() return n, nil } @@ -995,26 +1022,26 @@ func (r *Repo) BulkSetExtra(ctx context.Context, ids []int64, adifKey, value str if len(ids) == 0 { return 0, nil } - ph := make([]string, len(ids)) - args := make([]any, 0, len(ids)+2) + head := []any{} expr := `json_set(COALESCE(extras_json, '{}'), '$.` + adifKey + `', ?)` if value == "" { expr = `json_remove(COALESCE(extras_json, '{}'), '$.` + adifKey + `')` } else { - args = append(args, value) + head = append(head, value) } - args = append(args, db.NowISO()) - for i, id := range ids { - ph[i] = "?" - args = append(args, id) - } - res, err := r.db.ExecContext(ctx, - `UPDATE qso SET extras_json = `+expr+`, updated_at = ? WHERE id IN (`+strings.Join(ph, ",")+`)`, - args...) + head = append(head, db.NowISO()) + n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + args := append(append([]any{}, head...), idArgs...) + res, err := r.db.ExecContext(ctx, + `UPDATE qso SET extras_json = `+expr+`, updated_at = ? WHERE id IN (`+ph+`)`, args...) + if err != nil { + return 0, err + } + return res.RowsAffected() + }) if err != nil { - return 0, fmt.Errorf("bulk set extra %s: %w", adifKey, err) + return n, fmt.Errorf("bulk set extra %s: %w", adifKey, err) } - n, _ := res.RowsAffected() return n, nil } @@ -1026,20 +1053,19 @@ func (r *Repo) BulkSetFrequency(ctx context.Context, ids []int64, freqHz int64, if len(ids) == 0 { return 0, nil } - ph := make([]string, len(ids)) - args := make([]any, 0, len(ids)+3) - args = append(args, freqHz, band, db.NowISO()) - for i, id := range ids { - ph[i] = "?" - args = append(args, id) - } - res, err := r.db.ExecContext(ctx, - `UPDATE qso SET freq_hz = ?, band = ?, updated_at = ? WHERE id IN (`+strings.Join(ph, ",")+`)`, - args...) + now := db.NowISO() + n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + args := append([]any{freqHz, band, now}, idArgs...) + res, err := r.db.ExecContext(ctx, + `UPDATE qso SET freq_hz = ?, band = ?, updated_at = ? WHERE id IN (`+ph+`)`, args...) + if err != nil { + return 0, err + } + return res.RowsAffected() + }) if err != nil { - return 0, fmt.Errorf("bulk set frequency: %w", err) + return n, fmt.Errorf("bulk set frequency: %w", err) } - n, _ := res.RowsAffected() return n, nil } @@ -1201,17 +1227,16 @@ func (r *Repo) DeleteMany(ctx context.Context, ids []int64) (int64, error) { if len(ids) == 0 { return 0, nil } - ph := make([]string, len(ids)) - args := make([]any, len(ids)) - for i, id := range ids { - ph[i] = "?" - args[i] = id - } - res, err := r.db.ExecContext(ctx, `DELETE FROM qso WHERE id IN (`+strings.Join(ph, ",")+`)`, args...) + n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + res, err := r.db.ExecContext(ctx, `DELETE FROM qso WHERE id IN (`+ph+`)`, idArgs...) + if err != nil { + return 0, err + } + return res.RowsAffected() + }) if err != nil { - return 0, fmt.Errorf("delete qsos: %w", err) + return n, fmt.Errorf("delete qsos: %w", err) } - n, _ := res.RowsAffected() return n, nil } @@ -1701,27 +1726,42 @@ func (r *Repo) IterateByIDs(ctx context.Context, ids []int64, fn func(QSO) error if len(ids) == 0 { return nil } - ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") - args := make([]any, len(ids)) - for i, id := range ids { - args[i] = id - } - rows, err := r.db.QueryContext(ctx, - `SELECT `+selectCols+` FROM qso WHERE id IN (`+ph+`) ORDER BY qso_date ASC, id ASC`, args...) + // Chunked like every other by-ids statement (the one-placeholder-per-id IN + // died at 168k with "too many SQL variables") — and because each chunk is + // only locally ordered, the rows are collected and sorted once at the end + // so the chronological contract holds across chunks. + var all []QSO + _, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) { + rows, err := r.db.QueryContext(ctx, + `SELECT `+selectCols+` FROM qso WHERE id IN (`+ph+`)`, idArgs...) + if err != nil { + return 0, err + } + defer rows.Close() + for rows.Next() { + q, err := scanQSO(rows) + if err != nil { + return 0, err + } + all = append(all, q) + } + return 0, rows.Err() + }) if err != nil { return fmt.Errorf("query qso: %w", err) } - defer rows.Close() - for rows.Next() { - q, err := scanQSO(rows) - if err != nil { - return err + sort.Slice(all, func(i, j int) bool { + if !all[i].QSODate.Equal(all[j].QSODate) { + return all[i].QSODate.Before(all[j].QSODate) } + return all[i].ID < all[j].ID + }) + for _, q := range all { if err := fn(q); err != nil { return err } } - return rows.Err() + return nil } // GridKey builds the lookup key for the worked-grid index.