fix(qso): every by-ids statement is chunked — 168k QSOs is a selection too

One placeholder per id in a single IN (…) hits SQLite's bound-variable cap: bulk field set died at 168 000 QSOs with 'too many SQL variables' (10 000 passed). One chunker now serves bulk set (text/int/extra/frequency), the delete, the post-upload markers and the export-selection iterator — the last collecting and sorting once at the end so its chronological contract holds across chunks. 500 ids a statement keeps every backend far from any limit.
2026-08-30 13:13:22 +02:00
parent 5a77fdf68f
commit 37298afd77
2 changed files with 129 additions and 79 deletions
@@ -1,4 +1,14 @@
 [
+  {
+    "version": "0.27.2",
+    "date": "",
+    "en": [
+      "Bulk operations work on any size of selection — setting a field, fixing frequencies, deleting, marking uploads and exporting the selection all failed with “too many SQL variables” past a few tens of thousands of QSOs. Statements are now issued in slices."
+    ],
+    "fr": [
+      "Les opérations groupées fonctionnent quelle que soit la taille de la sélection — définir un champ, corriger des fréquences, supprimer, marquer les uploads et exporter la sélection échouaient avec « too many SQL variables » au-delà de quelques dizaines de milliers de QSO. Les requêtes sont désormais émises par tranches."
+    ]
+  },
  {
    "version": "0.27.1",
    "date": "",
@@ -722,15 +722,14 @@ func (r *Repo) MarkUploadedBatch(ctx context.Context, statusCol, dateCol, date s
 	if len(ids) == 0 {
 		return nil
 	}
-	ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
-	args := make([]any, 0, len(ids)+2)
-	args = append(args, date, db.NowISO())
-	for _, id := range ids {
-		args = append(args, id)
-	}
+	now := db.NowISO()
+	_, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		args := append([]any{date, now}, idArgs...)
 		_, err := r.db.ExecContext(ctx,
 			`UPDATE qso SET `+statusCol+` = 'Y', `+dateCol+` = ?, updated_at = ? WHERE id IN (`+ph+`)`,
 			args...)
+		return 0, err
+	})
 	if err != nil {
 		return fmt.Errorf("mark uploaded batch (%d): %w", len(ids), err)
 	}
@@ -872,6 +871,35 @@ var bulkEditableCols = map[string]bool{
 // own path, not the text one: the columns are nullable integers, and while
 // SQLite would coerce "14" quietly, a shared MySQL logbook would not — and an
 // empty string is NULL here, never "".
+// bulkByIDChunks runs one UPDATE per slice of ids, small enough for SQLite's
+// bound-variable cap: the single IN (…) with one placeholder per id worked at
+// 10 000 QSOs and failed at 168 000 with "too many SQL variables". Each call
+// gets the placeholder string and the id arguments for its slice; affected
+// rows are summed. 500 per statement keeps every backend far from any limit
+// while costing a few hundred statements on the largest logs.
+func bulkByIDChunks(ctx context.Context, ids []int64, run func(ph string, idArgs []any) (int64, error)) (int64, error) {
+	const chunk = 500
+	var total int64
+	for start := 0; start < len(ids); start += chunk {
+		end := start + chunk
+		if end > len(ids) {
+			end = len(ids)
+		}
+		part := ids[start:end]
+		ph := strings.Repeat("?,", len(part)-1) + "?"
+		args := make([]any, len(part))
+		for i, id := range part {
+			args[i] = id
+		}
+		n, err := run(ph, args)
+		if err != nil {
+			return total, err
+		}
+		total += n
+	}
+	return total, nil
+}
+
 var bulkEditableIntCols = map[string]bool{
 	"my_dxcc":     true,
 	"my_cq_zone":  true,
@@ -886,23 +914,23 @@ func (r *Repo) BulkSetIntField(ctx context.Context, ids []int64, column string,
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	ph := make([]string, len(ids))
-	args := make([]any, 0, len(ids)+2)
 	var val any
 	if v != nil {
 		val = *v
 	}
-	args = append(args, val, db.NowISO())
-	for i, id := range ids {
-		ph[i] = "?"
-		args = append(args, id)
-	}
+	now := db.NowISO()
+	n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		args := append([]any{val, now}, idArgs...)
 		res, err := r.db.ExecContext(ctx,
-		"UPDATE qso SET "+column+" = ?, updated_at = ? WHERE id IN ("+strings.Join(ph, ",")+")", args...)
+			"UPDATE qso SET "+column+" = ?, updated_at = ? WHERE id IN ("+ph+")", args...)
 		if err != nil {
-		return 0, fmt.Errorf("bulk set %s: %w", column, err)
+			return 0, err
+		}
+		return res.RowsAffected()
+	})
+	if err != nil {
+		return n, fmt.Errorf("bulk set %s: %w", column, err)
 	}
-	n, _ := res.RowsAffected()
 	return n, nil
 }

@@ -913,13 +941,6 @@ func (r *Repo) BulkSetField(ctx context.Context, ids []int64, column, value stri
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	ph := make([]string, len(ids))
-	args := make([]any, 0, len(ids)+2)
-	args = append(args, value, db.NowISO())
-	for i, id := range ids {
-		ph[i] = "?"
-		args = append(args, id)
-	}
 	set := column + " = ?, updated_at = ?"
 	if column == "mode" {
 		// A submode belongs to the mode it was recorded under. Left behind, it
@@ -928,13 +949,19 @@ func (r *Repo) BulkSetField(ctx context.Context, ids []int64, column, value stri
 		// only outcome that leaves the row meaning what the operator asked for.
 		set += ", submode = ''"
 	}
+	now := db.NowISO()
+	n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		args := append([]any{value, now}, idArgs...)
 		res, err := r.db.ExecContext(ctx,
-		`UPDATE qso SET `+set+` WHERE id IN (`+strings.Join(ph, ",")+`)`,
-		args...)
+			`UPDATE qso SET `+set+` WHERE id IN (`+ph+`)`, args...)
 		if err != nil {
-		return 0, fmt.Errorf("bulk set %s: %w", column, err)
+			return 0, err
+		}
+		return res.RowsAffected()
+	})
+	if err != nil {
+		return n, fmt.Errorf("bulk set %s: %w", column, err)
 	}
-	n, _ := res.RowsAffected()
 	return n, nil
 }

@@ -995,26 +1022,26 @@ func (r *Repo) BulkSetExtra(ctx context.Context, ids []int64, adifKey, value str
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	ph := make([]string, len(ids))
-	args := make([]any, 0, len(ids)+2)
+	head := []any{}
 	expr := `json_set(COALESCE(extras_json, '{}'), '$.` + adifKey + `', ?)`
 	if value == "" {
 		expr = `json_remove(COALESCE(extras_json, '{}'), '$.` + adifKey + `')`
 	} else {
-		args = append(args, value)
-	}
-	args = append(args, db.NowISO())
-	for i, id := range ids {
-		ph[i] = "?"
-		args = append(args, id)
+		head = append(head, value)
 	}
+	head = append(head, db.NowISO())
+	n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		args := append(append([]any{}, head...), idArgs...)
 		res, err := r.db.ExecContext(ctx,
-		`UPDATE qso SET extras_json = `+expr+`, updated_at = ? WHERE id IN (`+strings.Join(ph, ",")+`)`,
-		args...)
+			`UPDATE qso SET extras_json = `+expr+`, updated_at = ? WHERE id IN (`+ph+`)`, args...)
 		if err != nil {
-		return 0, fmt.Errorf("bulk set extra %s: %w", adifKey, err)
+			return 0, err
+		}
+		return res.RowsAffected()
+	})
+	if err != nil {
+		return n, fmt.Errorf("bulk set extra %s: %w", adifKey, err)
 	}
-	n, _ := res.RowsAffected()
 	return n, nil
 }

@@ -1026,20 +1053,19 @@ func (r *Repo) BulkSetFrequency(ctx context.Context, ids []int64, freqHz int64,
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	ph := make([]string, len(ids))
-	args := make([]any, 0, len(ids)+3)
-	args = append(args, freqHz, band, db.NowISO())
-	for i, id := range ids {
-		ph[i] = "?"
-		args = append(args, id)
-	}
+	now := db.NowISO()
+	n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		args := append([]any{freqHz, band, now}, idArgs...)
 		res, err := r.db.ExecContext(ctx,
-		`UPDATE qso SET freq_hz = ?, band = ?, updated_at = ? WHERE id IN (`+strings.Join(ph, ",")+`)`,
-		args...)
+			`UPDATE qso SET freq_hz = ?, band = ?, updated_at = ? WHERE id IN (`+ph+`)`, args...)
 		if err != nil {
-		return 0, fmt.Errorf("bulk set frequency: %w", err)
+			return 0, err
+		}
+		return res.RowsAffected()
+	})
+	if err != nil {
+		return n, fmt.Errorf("bulk set frequency: %w", err)
 	}
-	n, _ := res.RowsAffected()
 	return n, nil
 }

@@ -1201,17 +1227,16 @@ func (r *Repo) DeleteMany(ctx context.Context, ids []int64) (int64, error) {
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	ph := make([]string, len(ids))
-	args := make([]any, len(ids))
-	for i, id := range ids {
-		ph[i] = "?"
-		args[i] = id
-	}
-	res, err := r.db.ExecContext(ctx, `DELETE FROM qso WHERE id IN (`+strings.Join(ph, ",")+`)`, args...)
+	n, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
+		res, err := r.db.ExecContext(ctx, `DELETE FROM qso WHERE id IN (`+ph+`)`, idArgs...)
 		if err != nil {
-		return 0, fmt.Errorf("delete qsos: %w", err)
+			return 0, err
+		}
+		return res.RowsAffected()
+	})
+	if err != nil {
+		return n, fmt.Errorf("delete qsos: %w", err)
 	}
-	n, _ := res.RowsAffected()
 	return n, nil
 }

@@ -1701,27 +1726,42 @@ func (r *Repo) IterateByIDs(ctx context.Context, ids []int64, fn func(QSO) error
 	if len(ids) == 0 {
 		return nil
 	}
-	ph := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
-	args := make([]any, len(ids))
-	for i, id := range ids {
-		args[i] = id
-	}
+	// Chunked like every other by-ids statement (the one-placeholder-per-id IN
+	// died at 168k with "too many SQL variables") — and because each chunk is
+	// only locally ordered, the rows are collected and sorted once at the end
+	// so the chronological contract holds across chunks.
+	var all []QSO
+	_, err := bulkByIDChunks(ctx, ids, func(ph string, idArgs []any) (int64, error) {
 		rows, err := r.db.QueryContext(ctx,
-		`SELECT `+selectCols+` FROM qso WHERE id IN (`+ph+`) ORDER BY qso_date ASC, id ASC`, args...)
+			`SELECT `+selectCols+` FROM qso WHERE id IN (`+ph+`)`, idArgs...)
 		if err != nil {
-		return fmt.Errorf("query qso: %w", err)
+			return 0, err
 		}
 		defer rows.Close()
 		for rows.Next() {
 			q, err := scanQSO(rows)
 			if err != nil {
-			return err
+				return 0, err
 			}
+			all = append(all, q)
+		}
+		return 0, rows.Err()
+	})
+	if err != nil {
+		return fmt.Errorf("query qso: %w", err)
+	}
+	sort.Slice(all, func(i, j int) bool {
+		if !all[i].QSODate.Equal(all[j].QSODate) {
+			return all[i].QSODate.Before(all[j].QSODate)
+		}
+		return all[i].ID < all[j].ID
+	})
+	for _, q := range all {
 		if err := fn(q); err != nil {
 			return err
 		}
 	}
-	return rows.Err()
+	return nil
 }

 // GridKey builds the lookup key for the worked-grid index.