up

2026-06-14 00:55:27 +02:00
parent 08162fa126
commit 67203cd4a8
16 changed files with 897 additions and 212 deletions
@@ -81,6 +81,34 @@ func PingMySQL(c MySQLConfig) error {
 //go:embed migrations/*.sql
 var migrationsFS embed.FS

+// Dialect names the backend opened at startup. Repos consult it for the few
+// places where SQLite and MySQL SQL genuinely differ (upserts, JSON extraction).
+// Timestamps are generated in Go (NowISO) for both, so most queries are shared.
+var Dialect = "sqlite"
+
+// IsMySQL reports whether the active LOGBOOK backend is the shared MySQL
+// server. Config tables always live in local SQLite, so this only governs
+// qso-table SQL (see columnExpr).
+func IsMySQL() bool { return Dialect == "mysql" }
+
+// SetDialect pins the logbook dialect ("mysql" | "sqlite"). Called once at
+// startup after the logbook backend is chosen, so it doesn't depend on the
+// order in which the local and logbook connections were opened.
+func SetDialect(d string) {
+	if d == "mysql" {
+		Dialect = "mysql"
+	} else {
+		Dialect = "sqlite"
+	}
+}
+
+// NowISO returns the current UTC time as the ISO-8601 string OpsLog stores in
+// *_at / date columns — millisecond precision, matching the old SQLite
+// strftime('%Y-%m-%dT%H:%M:%fZ','now') default. Bind this as a parameter so the
+// same INSERT/UPDATE works on both backends.
+func NowISO() string { return time.Now().UTC().Format("2006-01-02T15:04:05.000Z") }
+
+
 // Open opens (and creates if needed) the SQLite database at the given path,
 // enables performance PRAGMAs, and applies embedded migrations.
 func Open(path string) (*sql.DB, error) {
@@ -97,7 +125,8 @@ func Open(path string) (*sql.DB, error) {
 		_ = conn.Close()
 		return nil, fmt.Errorf("ping sqlite: %w", err)
 	}
-	if err := migrate(conn); err != nil {
+	Dialect = "sqlite"
+	if err := migrate(conn, nil); err != nil {
 		_ = conn.Close()
 		return nil, err
 	}
@@ -106,12 +135,16 @@ func Open(path string) (*sql.DB, error) {

 // migrate applies all embedded *.sql migrations in alphabetical order,
 // skipping those already applied. Intentionally minimal in-house system
-// (no external dependency).
-func migrate(conn *sql.DB) error {
-	if _, err := conn.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
+// (no external dependency). translate, when non-nil, rewrites each statement
+// for a non-SQLite backend (see mysqlDDL); nil means run the SQLite DDL as-is.
+func migrate(conn *sql.DB, translate func(string) string) error {
+	if translate == nil {
+		translate = func(s string) string { return s }
+	}
+	if _, err := conn.Exec(translate(`CREATE TABLE IF NOT EXISTS schema_migrations (
        name TEXT PRIMARY KEY,
        applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
-    )`); err != nil {
+    )`)); err != nil {
 		return fmt.Errorf("create schema_migrations: %w", err)
 	}

@@ -141,11 +174,30 @@ func migrate(conn *sql.DB) error {
 		if err != nil {
 			return fmt.Errorf("read migration %s: %w", name, err)
 		}
+		sqlText := translate(string(content))
+
+		// MySQL implicitly commits each DDL statement, so a wrapping transaction
+		// gives no atomicity — a mid-file failure would leave columns/tables
+		// behind, unrecorded, and every restart would re-run and choke on
+		// "duplicate column". Instead apply statement-by-statement and tolerate
+		// "already exists" errors, making migrations idempotent (and self-healing
+		// for a previously half-applied database).
+		if IsMySQL() {
+			if err := applyMySQLMigration(conn, sqlText); err != nil {
+				return fmt.Errorf("apply migration %s: %w", name, err)
+			}
+			if _, err := conn.Exec(`INSERT INTO schema_migrations(name) VALUES(?)`, name); err != nil {
+				return fmt.Errorf("record migration %s: %w", name, err)
+			}
+			continue
+		}
+
+		// SQLite supports DDL inside a transaction, so keep it atomic there.
 		tx, err := conn.Begin()
 		if err != nil {
 			return fmt.Errorf("begin tx for %s: %w", name, err)
 		}
-		if _, err := tx.Exec(string(content)); err != nil {
+		if _, err := tx.Exec(sqlText); err != nil {
 			_ = tx.Rollback()
 			return fmt.Errorf("apply migration %s: %w", name, err)
 		}
@@ -0,0 +1,242 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	mysqldriver "github.com/go-sql-driver/mysql"
+)
+
+// ── SQLite → MySQL schema translation ──────────────────────────────────
+//
+// OpsLog's migrations are authored in SQLite dialect (the default backend).
+// For the shared-MySQL backend we translate that DDL on the fly rather than
+// maintaining a second set of migration files, so the two backends can never
+// drift apart. The translation is deliberately narrow — it only rewrites the
+// SQLite-isms that actually appear in our migrations:
+//
+//   - INTEGER PRIMARY KEY [AUTOINCREMENT]  → BIGINT AUTO_INCREMENT PRIMARY KEY
+//   - bare INTEGER                          → BIGINT (so FK child/parent types match)
+//   - DEFAULT (strftime(... 'now'))         → DEFAULT '' (timestamps come from Go)
+//   - TEXT                                  → VARCHAR(255), except a few free-text
+//                                             columns that stay TEXT/LONGTEXT
+//   - reserved column `key`                 → backticked
+//   - CREATE INDEX IF NOT EXISTS            → CREATE INDEX (MySQL has no IF NOT EXISTS)
+//
+// REAL is left as-is (MySQL accepts it as an alias for DOUBLE). CREATE/DROP
+// TABLE IF [NOT] EXISTS, RENAME TO, INSERT…SELECT, CHECK(...) and foreign keys
+// are all valid MySQL as written.
+
+var (
+	reStrftimeDefault = regexp.MustCompile(`\(strftime\('%Y-%m-%dT%H:%M:%fZ',\s*'now'\)\)`)
+	reBareInteger     = regexp.MustCompile(`\bINTEGER\b`)
+	reColText         = regexp.MustCompile(`(\w+)\s+TEXT\b`)
+	reKeyColumn       = regexp.MustCompile(`(?m)^(\s*)key\b`)
+)
+
+// MySQL's InnoDB row-size limit is 65535 bytes (excluding off-page TEXT/BLOB).
+// The qso table has ~150 columns, so making them all VARCHAR(255) (1020 bytes
+// each in utf8mb4) overflows the row. We therefore emit TEXT (stored off-page,
+// ~20 bytes in-row) for the bulk of columns, and VARCHAR only where a column
+// actually needs in-row storage: it's indexed, a primary key, or carries a
+// DEFAULT (TEXT can't hold a literal default in MySQL).
+
+// varcharColumns must be VARCHAR because they're indexed or part of a primary
+// key declared on a separate line (so the DEFAULT/PRIMARY-KEY line heuristic
+// below can't catch them). MySQL can't index a TEXT column without a prefix
+// length. Keyed by column name (a name may repeat across tables — harmless).
+var varcharColumns = map[string]bool{
+	// qso indexes (0001, 0003, 0019)
+	"callsign": true, "qso_date": true, "band": true, "mode": true,
+	"grid": true, "station_callsign": true, "state": true, "contest_id": true,
+	"sat_name": true, "prop_mode": true, "sig": true, "wwff_ref": true, "skcc": true,
+	// integrations_udp index (0011)
+	"direction": true,
+	// award_references composite primary key (0017)
+	"award_code": true, "ref_code": true,
+}
+
+// longTextColumns override the default TEXT with a bigger type where the
+// content can be large (a full QSL-template document).
+var longTextColumns = map[string]string{
+	"json": "LONGTEXT",
+}
+
+// mysqlDDL rewrites a SQLite DDL/DML statement (or multi-statement migration
+// file) into the MySQL dialect. See the package note above for the rules.
+func mysqlDDL(stmt string) string {
+	s := stmt
+	// MySQL has no IF NOT EXISTS for CREATE INDEX (migrations run once anyway).
+	s = strings.ReplaceAll(s, "CREATE INDEX IF NOT EXISTS", "CREATE INDEX")
+	// Auto-increment primary keys. Both the AUTOINCREMENT form and the bare
+	// "INTEGER PRIMARY KEY" (SQLite rowid alias) must become AUTO_INCREMENT so
+	// inserts that omit the id still get one.
+	s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY AUTOINCREMENT", "BIGINT AUTO_INCREMENT PRIMARY KEY")
+	s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY", "BIGINT AUTO_INCREMENT PRIMARY KEY")
+	// Timestamp defaults: Go supplies ISO strings, so drop the SQLite function
+	// default (TEXT/VARCHAR can't carry it in MySQL anyway).
+	s = reStrftimeDefault.ReplaceAllString(s, "''")
+	// Remaining INTEGER columns → BIGINT so foreign-key parent/child types match.
+	s = reBareInteger.ReplaceAllString(s, "BIGINT")
+	// TEXT columns → TEXT (off-page) by default, VARCHAR(255) only when indexed,
+	// a primary key, or default-bearing — keeping the row under MySQL's limit.
+	s = translateTextColumns(s)
+	// `key` is a MySQL reserved word — backtick the settings column declaration.
+	s = reKeyColumn.ReplaceAllString(s, "$1`key`")
+	return s
+}
+
+// translateTextColumns rewrites each "<col> TEXT" column declaration to the
+// right MySQL type, deciding per line so it can see whether the line carries a
+// DEFAULT or an inline PRIMARY KEY. See varcharColumns / longTextColumns.
+func translateTextColumns(s string) string {
+	lines := strings.Split(s, "\n")
+	for i, line := range lines {
+		m := reColText.FindStringSubmatchIndex(line)
+		if m == nil {
+			continue
+		}
+		col := line[m[2]:m[3]]
+		var typ string
+		switch {
+		case longTextColumns[col] != "":
+			typ = longTextColumns[col]
+		case varcharColumns[col],
+			strings.Contains(line, "DEFAULT"),
+			strings.Contains(line, "PRIMARY KEY"):
+			typ = "VARCHAR(255)"
+		default:
+			typ = "TEXT"
+		}
+		lines[i] = line[:m[0]] + col + " " + typ + line[m[1]:]
+	}
+	return strings.Join(lines, "\n")
+}
+
+// OpenMySQL opens the shared MySQL logbook, creating the database if needed,
+// then applies the (translated) embedded migrations. multiStatements is enabled
+// so multi-statement migration files run in a single Exec.
+func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
+	if strings.TrimSpace(c.Host) == "" {
+		return nil, fmt.Errorf("host is required")
+	}
+	name := strings.TrimSpace(c.Database)
+	if !validDBIdent(name) {
+		return nil, fmt.Errorf("invalid database name %q (letters, digits, underscore only)", name)
+	}
+	// Ensure the database exists (connect server-level first).
+	if err := PingMySQL(c); err != nil {
+		return nil, err
+	}
+	conn, err := sql.Open("mysql", c.dsn())
+	if err != nil {
+		return nil, fmt.Errorf("open mysql: %w", err)
+	}
+	// The UI fires bursts of concurrent queries (the Preferences dialog alone
+	// loads ~8 settings in parallel, plus the grid and live cluster status). A
+	// low cap turns such a burst into a deadlock-like stall against a remote
+	// server, so keep a generous pool with idle reaping. SQLite ran uncapped.
+	conn.SetMaxOpenConns(50)
+	conn.SetMaxIdleConns(10)
+	conn.SetConnMaxIdleTime(90 * time.Second)
+	conn.SetConnMaxLifetime(5 * time.Minute)
+	if err := conn.Ping(); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("connect to %s: %w", name, err)
+	}
+	// Set the dialect before migrating so the runner takes the MySQL path
+	// (per-statement, idempotent) rather than the SQLite transaction path.
+	Dialect = "mysql"
+	if err := migrate(conn, mysqlDDL); err != nil {
+		_ = conn.Close()
+		Dialect = "sqlite" // migration failed; we'll fall back to SQLite
+		return nil, err
+	}
+	return conn, nil
+}
+
+// applyMySQLMigration executes a translated migration one statement at a time.
+// MySQL has no multi-statement Exec without a special flag and auto-commits DDL,
+// so running statements individually (and tolerating "already exists" errors)
+// keeps migrations idempotent and lets a half-applied database self-heal.
+//
+// It runs on a single dedicated connection with FOREIGN_KEY_CHECKS=0, because a
+// few migrations recreate tables by DROP/RENAME in an order that trips MySQL's
+// foreign-key enforcement (SQLite is laxer). The flag is reset to 1 before the
+// connection returns to the pool, so runtime queries keep FK enforcement.
+func applyMySQLMigration(conn *sql.DB, sqlText string) error {
+	ctx := context.Background()
+	c, err := conn.Conn(ctx)
+	if err != nil {
+		return fmt.Errorf("acquire conn: %w", err)
+	}
+	defer func() {
+		_, _ = c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=1")
+		_ = c.Close()
+	}()
+	if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil {
+		return fmt.Errorf("disable fk checks: %w", err)
+	}
+	for _, stmt := range splitStatements(sqlText) {
+		if _, err := c.ExecContext(ctx, stmt); err != nil {
+			if isIgnorableDDLError(err) {
+				continue
+			}
+			return fmt.Errorf("statement %q: %w", firstLine(stmt), err)
+		}
+	}
+	return nil
+}
+
+// splitStatements breaks a migration file into individual SQL statements,
+// dropping full-line comments and blank fragments. Our migrations never embed
+// a ';' inside a string literal, so a simple split is safe.
+func splitStatements(sqlText string) []string {
+	var b strings.Builder
+	for _, line := range strings.Split(sqlText, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(line), "--") {
+			continue
+		}
+		b.WriteString(line)
+		b.WriteByte('\n')
+	}
+	var out []string
+	for _, part := range strings.Split(b.String(), ";") {
+		if strings.TrimSpace(part) != "" {
+			out = append(out, part)
+		}
+	}
+	return out
+}
+
+func firstLine(s string) string {
+	s = strings.TrimSpace(s)
+	if i := strings.IndexByte(s, '\n'); i >= 0 {
+		return s[:i]
+	}
+	return s
+}
+
+// isIgnorableDDLError reports whether a MySQL error means the object the
+// statement creates/drops is already in the intended state — safe to skip when
+// re-applying a forward-only migration.
+func isIgnorableDDLError(err error) bool {
+	var me *mysqldriver.MySQLError
+	if !errors.As(err, &me) {
+		return false
+	}
+	switch me.Number {
+	case 1050, // ER_TABLE_EXISTS_ERROR — CREATE TABLE on an existing table
+		1051, // ER_BAD_TABLE_ERROR — DROP TABLE on a missing table
+		1060, // ER_DUP_FIELDNAME — ADD COLUMN already present
+		1061, // ER_DUP_KEYNAME — CREATE INDEX already present
+		1091: // ER_CANT_DROP_FIELD_OR_KEY — DROP of something that doesn't exist
+		return true
+	}
+	return false
+}
@@ -0,0 +1,153 @@
+package db
+
+import (
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func TestMySQLDDL_Translations(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{"autoinc pk", "id INTEGER PRIMARY KEY AUTOINCREMENT,", "id BIGINT AUTO_INCREMENT PRIMARY KEY,"},
+		{"plain pk", "id INTEGER PRIMARY KEY,", "id BIGINT AUTO_INCREMENT PRIMARY KEY,"},
+		{"bare integer", "freq_hz INTEGER,", "freq_hz BIGINT,"},
+		{"strftime default", "created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),",
+			"created_at VARCHAR(255) NOT NULL DEFAULT '',"},
+		{"strftime tight", "updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),",
+			"updated_at VARCHAR(255) NOT NULL DEFAULT '',"},
+		// The column-name/type whitespace collapses to a single space — harmless.
+		{"text default N", "qsl_sent        TEXT DEFAULT 'N',", "qsl_sent VARCHAR(255) DEFAULT 'N',"},
+		{"indexed col → varchar", "callsign        TEXT    NOT NULL,", "callsign VARCHAR(255)    NOT NULL,"},
+		{"plain non-indexed col → text", "name            TEXT,", "name TEXT,"},
+		{"plain text stays text", "comment         TEXT,", "comment TEXT,"},
+		{"json longtext", "    json        TEXT NOT NULL,", "    json LONGTEXT NOT NULL,"},
+		{"create index", "CREATE INDEX IF NOT EXISTS idx_qso_dxcc ON qso(dxcc);",
+			"CREATE INDEX idx_qso_dxcc ON qso(dxcc);"},
+	}
+	for _, c := range cases {
+		if got := mysqlDDL(c.in); got != c.want {
+			t.Errorf("%s:\n  in   %q\n  got  %q\n  want %q", c.name, c.in, got, c.want)
+		}
+	}
+}
+
+func TestMySQLDDL_KeyColumnBackticked(t *testing.T) {
+	in := "CREATE TABLE IF NOT EXISTS settings (\n    key        TEXT PRIMARY KEY,\n    value      TEXT NOT NULL\n);"
+	got := mysqlDDL(in)
+	if !strings.Contains(got, "`key` VARCHAR(255) PRIMARY KEY") {
+		t.Errorf("key column not backticked/translated:\n%s", got)
+	}
+}
+
+func TestSplitStatements(t *testing.T) {
+	in := "-- a comment\n" +
+		"ALTER TABLE qso ADD COLUMN a TEXT;\n" +
+		"ALTER TABLE qso ADD COLUMN b TEXT;  -- inline note\n" +
+		"\n" +
+		"CREATE INDEX idx ON qso(a);\n"
+	got := splitStatements(in)
+	if len(got) != 3 {
+		t.Fatalf("want 3 statements, got %d: %#v", len(got), got)
+	}
+	if !strings.Contains(got[0], "ADD COLUMN a") ||
+		!strings.Contains(got[1], "ADD COLUMN b") ||
+		!strings.Contains(got[2], "CREATE INDEX") {
+		t.Errorf("unexpected split: %#v", got)
+	}
+	// No fragment should be a comment-only or blank statement.
+	for _, s := range got {
+		if strings.TrimSpace(s) == "" {
+			t.Errorf("empty statement in result: %#v", got)
+		}
+	}
+}
+
+// Every embedded migration must split into at least one runnable statement
+// and never produce an empty fragment.
+func TestSplitStatements_AllMigrations(t *testing.T) {
+	entries, _ := migrationsFS.ReadDir("migrations")
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".sql") {
+			continue
+		}
+		raw, _ := migrationsFS.ReadFile("migrations/" + e.Name())
+		stmts := splitStatements(mysqlDDL(string(raw)))
+		if len(stmts) == 0 {
+			t.Errorf("%s: produced no statements", e.Name())
+		}
+		for _, s := range stmts {
+			if strings.TrimSpace(s) == "" {
+				t.Errorf("%s: empty statement fragment", e.Name())
+			}
+		}
+	}
+}
+
+// TestMySQLDDL_QSORowSizeUnderLimit guards against the InnoDB 65535-byte row
+// limit: every VARCHAR(255) in utf8mb4 costs 1020 bytes in-row, and the qso
+// table (built from 0001 + the qso-only ALTERs in 0003 and 0019) must stay well
+// clear of it. TEXT columns are off-page and don't count here.
+func TestMySQLDDL_QSORowSizeUnderLimit(t *testing.T) {
+	qsoMigrations := []string{"0001_init.sql", "0003_adif_extra.sql", "0019_adif_317_fields.sql"}
+	varchars := 0
+	for _, name := range qsoMigrations {
+		raw, err := migrationsFS.ReadFile("migrations/" + name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		varchars += strings.Count(mysqlDDL(string(raw)), "VARCHAR(255)")
+	}
+	const bytesPerVarchar = 255 * 4 // utf8mb4
+	rowBytes := varchars * bytesPerVarchar
+	t.Logf("qso VARCHAR(255) columns: %d (~%d bytes in-row)", varchars, rowBytes)
+	if rowBytes > 60000 { // leave headroom under the 65535 hard limit
+		t.Errorf("qso row too large: %d VARCHAR(255) cols = ~%d bytes (limit 65535)", varchars, rowBytes)
+	}
+}
+
+// TestMySQLDDL_NoLeftoverSQLiteisms translates every embedded migration and
+// fails if any SQLite-only construct survives — a fast guard against a new
+// migration sneaking in a dialect-ism the translator doesn't cover.
+func TestMySQLDDL_NoLeftoverSQLiteisms(t *testing.T) {
+	entries, err := migrationsFS.ReadDir("migrations")
+	if err != nil {
+		t.Fatal(err)
+	}
+	reInteger := regexp.MustCompile(`\bINTEGER\b`)
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".sql") {
+			continue
+		}
+		raw, err := migrationsFS.ReadFile("migrations/" + e.Name())
+		if err != nil {
+			t.Fatal(err)
+		}
+		out := mysqlDDL(string(raw))
+		if strings.Contains(out, "AUTOINCREMENT") {
+			t.Errorf("%s: AUTOINCREMENT survived", e.Name())
+		}
+		if strings.Contains(out, "strftime") {
+			t.Errorf("%s: strftime survived", e.Name())
+		}
+		if strings.Contains(out, "IF NOT EXISTS idx") {
+			t.Errorf("%s: CREATE INDEX IF NOT EXISTS survived", e.Name())
+		}
+		// Strip comment lines before checking for bare INTEGER (comments are
+		// prose and may legitimately mention the word).
+		var code strings.Builder
+		for _, ln := range strings.Split(out, "\n") {
+			if strings.HasPrefix(strings.TrimSpace(ln), "--") {
+				continue
+			}
+			code.WriteString(ln)
+			code.WriteByte('\n')
+		}
+		if reInteger.MatchString(code.String()) {
+			t.Errorf("%s: bare INTEGER survived in code", e.Name())
+		}
+	}
+}