up

2026-06-14 00:55:27 +02:00
parent 08162fa126
commit 67203cd4a8
16 changed files with 897 additions and 212 deletions
@@ -10,6 +10,8 @@ import (
 	"sort"
 	"strings"
 	"time"
+
+	"hamlog/internal/db"
 )

 // MergeNonZero copies every non-zero field of src onto dst. Zero-value src
@@ -255,6 +257,19 @@ var columnCount = countColumns(columnList)
 // insertPlaceholders returns "?,?,?,..." matching columnCount.
 var insertPlaceholders = buildInsertPlaceholders()

+// insertCols/insertVals append created_at + updated_at so they're set from Go
+// (NowISO) on every backend — MySQL has no strftime default, and binding the
+// timestamp keeps a single backend-agnostic INSERT. insertArgs pairs with them.
+const insertCols = columnList + `, created_at, updated_at`
+
+var insertVals = insertPlaceholders + ",?,?"
+
+// insertArgs returns the column values plus the two timestamps for an INSERT.
+func (q *QSO) insertArgs() []any {
+	now := db.NowISO()
+	return append(q.args(), now, now)
+}
+
 func countColumns(s string) int {
 	n := 1
 	for i := 0; i < len(s); i++ {
@@ -349,34 +364,30 @@ func (r *Repo) Add(ctx context.Context, q QSO) (int64, error) {
 		q.QSODate = time.Now().UTC()
 	}
 	res, err := r.db.ExecContext(ctx,
-		`INSERT INTO qso (`+columnList+`) VALUES (`+insertPlaceholders+`)`,
-		q.args()...)
+		`INSERT INTO qso (`+insertCols+`) VALUES (`+insertVals+`)`,
+		q.insertArgs()...)
 	if err != nil {
 		return 0, fmt.Errorf("insert qso: %w", err)
 	}
 	return res.LastInsertId()
 }

-// AddBatch inserts many QSOs inside a single transaction using a prepared
-// statement. Empty-callsign records are skipped. Returns rows inserted.
+// batchInsertRows is how many QSOs go into one multi-row INSERT on MySQL. Each
+// row carries ~135 columns, so 200 rows ≈ 27k bound parameters — well under
+// MySQL's 65535-placeholder limit and a modest packet — while cutting network
+// round-trips ~200× versus one INSERT per row (critical for a remote server).
+const batchInsertRows = 200
+
+// AddBatch inserts many QSOs inside a single transaction. Empty-callsign records
+// are skipped. On MySQL it uses chunked multi-row INSERTs so a 27k-record import
+// over a remote link takes seconds, not many minutes; on local SQLite a prepared
+// statement per row is already fast.
 func (r *Repo) AddBatch(ctx context.Context, qsos []QSO) (int64, error) {
 	if len(qsos) == 0 {
 		return 0, nil
 	}
-	tx, err := r.db.BeginTx(ctx, nil)
-	if err != nil {
-		return 0, fmt.Errorf("begin tx: %w", err)
-	}
-	defer tx.Rollback()
-
-	stmt, err := tx.PrepareContext(ctx,
-		`INSERT INTO qso (`+columnList+`) VALUES (`+insertPlaceholders+`)`)
-	if err != nil {
-		return 0, fmt.Errorf("prepare batch insert: %w", err)
-	}
-	defer stmt.Close()
-
-	var inserted int64
+	// Normalise and drop empty-callsign records up front.
+	rows := make([]QSO, 0, len(qsos))
 	for _, q := range qsos {
 		q.Callsign = strings.ToUpper(strings.TrimSpace(q.Callsign))
 		if q.Callsign == "" {
@@ -385,10 +396,57 @@ func (r *Repo) AddBatch(ctx context.Context, qsos []QSO) (int64, error) {
 		if q.QSODate.IsZero() {
 			q.QSODate = time.Now().UTC()
 		}
-		if _, err := stmt.ExecContext(ctx, q.args()...); err != nil {
-			return inserted, fmt.Errorf("insert qso %q: %w", q.Callsign, err)
+		rows = append(rows, q)
+	}
+	if len(rows) == 0 {
+		return 0, nil
+	}
+
+	tx, err := r.db.BeginTx(ctx, nil)
+	if err != nil {
+		return 0, fmt.Errorf("begin tx: %w", err)
+	}
+	defer tx.Rollback()
+
+	var inserted int64
+	if db.IsMySQL() {
+		rowPlaceholder := "(" + insertVals + ")"
+		for start := 0; start < len(rows); start += batchInsertRows {
+			end := start + batchInsertRows
+			if end > len(rows) {
+				end = len(rows)
+			}
+			chunk := rows[start:end]
+			var sb strings.Builder
+			sb.WriteString("INSERT INTO qso (")
+			sb.WriteString(insertCols)
+			sb.WriteString(") VALUES ")
+			args := make([]any, 0, len(chunk)*(columnCount+2))
+			for j := range chunk {
+				if j > 0 {
+					sb.WriteByte(',')
+				}
+				sb.WriteString(rowPlaceholder)
+				args = append(args, chunk[j].insertArgs()...)
+			}
+			if _, err := tx.ExecContext(ctx, sb.String(), args...); err != nil {
+				return inserted, fmt.Errorf("batch insert: %w", err)
+			}
+			inserted += int64(len(chunk))
+		}
+	} else {
+		stmt, err := tx.PrepareContext(ctx,
+			`INSERT INTO qso (`+insertCols+`) VALUES (`+insertVals+`)`)
+		if err != nil {
+			return 0, fmt.Errorf("prepare batch insert: %w", err)
+		}
+		defer stmt.Close()
+		for i := range rows {
+			if _, err := stmt.ExecContext(ctx, rows[i].insertArgs()...); err != nil {
+				return inserted, fmt.Errorf("insert qso %q: %w", rows[i].Callsign, err)
+			}
+			inserted++
 		}
-		inserted++
 	}
 	if err := tx.Commit(); err != nil {
 		return 0, fmt.Errorf("commit batch: %w", err)
@@ -455,8 +513,8 @@ func (r *Repo) ListForUpload(ctx context.Context, column, value string) ([]Uploa
 func (r *Repo) MarkQRZUploaded(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET qrzcom_qso_upload_status = 'Y', qrzcom_qso_upload_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark qrz uploaded %d: %w", id, err)
 	}
@@ -468,8 +526,8 @@ func (r *Repo) MarkQRZUploaded(ctx context.Context, id int64, date string) error
 func (r *Repo) MarkClublogUploaded(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET clublog_qso_upload_status = 'Y', clublog_qso_upload_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark clublog uploaded %d: %w", id, err)
 	}
@@ -481,8 +539,8 @@ func (r *Repo) MarkClublogUploaded(ctx context.Context, id int64, date string) e
 func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET lotw_sent = 'Y', lotw_sent_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark lotw uploaded %d: %w", id, err)
 	}
@@ -494,8 +552,8 @@ func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) erro
 func (r *Repo) MarkEQSLSent(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET eqsl_sent = 'Y', eqsl_sent_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark eqsl sent %d: %w", id, err)
 	}
@@ -515,9 +573,9 @@ func (r *Repo) Update(ctx context.Context, q QSO) error {
 		q.QSODate = time.Now().UTC()
 	}
 	setClause := buildUpdateSetClause()
-	args := append(q.args(), q.ID)
+	args := append(q.args(), db.NowISO(), q.ID)
 	res, err := r.db.ExecContext(ctx,
-		`UPDATE qso SET `+setClause+`, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
+		`UPDATE qso SET `+setClause+`, updated_at = ? WHERE id = ?`,
 		args...)
 	if err != nil {
 		return fmt.Errorf("update qso %d: %w", q.ID, err)
@@ -714,6 +772,11 @@ func columnExpr(field string) (string, bool) {
 		return f, true
 	}
 	if key, ok := filterableExtras[f]; ok {
+		if db.IsMySQL() {
+			// JSON_EXTRACT errors on an invalid/empty document, so guard with
+			// NULLIF; JSON_UNQUOTE strips the quotes MySQL adds around strings.
+			return "JSON_UNQUOTE(JSON_EXTRACT(NULLIF(extras_json,''), '$." + key + "'))", true
+		}
 		return "json_extract(extras_json, '$." + key + "')", true
 	}
 	return "", false
@@ -1348,7 +1411,7 @@ func (r *Repo) Count(ctx context.Context) (int64, error) {
 // far cheaper than N exists-queries during the import loop.
 func (r *Repo) ExistingDedupeKeys(ctx context.Context) (map[string]struct{}, error) {
 	rows, err := r.db.QueryContext(ctx, `
-		SELECT callsign, strftime('%Y-%m-%dT%H:%M', qso_date), band, mode
+		SELECT callsign, substr(qso_date, 1, 16), band, mode
 		FROM qso`)
 	if err != nil {
 		return nil, err
@@ -1376,7 +1439,7 @@ func DedupeKey(callsign, qsoDateMinute, band, mode string) string {
 // confirmations back to local QSOs.
 func (r *Repo) DedupeKeyIDs(ctx context.Context) (map[string]int64, error) {
 	rows, err := r.db.QueryContext(ctx, `
-		SELECT id, callsign, strftime('%Y-%m-%dT%H:%M', qso_date), band, mode
+		SELECT id, callsign, substr(qso_date, 1, 16), band, mode
 		FROM qso`)
 	if err != nil {
 		return nil, err
@@ -1463,8 +1526,8 @@ func (r *Repo) ConfirmedSlots(ctx context.Context, cols []string) (ConfirmedSets
 func (r *Repo) MarkQRZConfirmed(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET qrzcom_qso_download_status = 'Y', qrzcom_qso_download_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark qrz confirmed %d: %w", id, err)
 	}
@@ -1476,8 +1539,8 @@ func (r *Repo) MarkQRZConfirmed(ctx context.Context, id int64, date string) erro
 func (r *Repo) MarkLoTWConfirmed(ctx context.Context, id int64, date string) error {
 	_, err := r.db.ExecContext(ctx,
 		`UPDATE qso SET lotw_rcvd = 'Y', lotw_rcvd_date = ?,
-		   updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`,
-		date, id)
+		   updated_at = ? WHERE id = ?`,
+		date, db.NowISO(), id)
 	if err != nil {
 		return fmt.Errorf("mark lotw confirmed %d: %w", id, err)
 	}