up

internal/db/db.go

@@ -29,8 +29,12 @@ func (c MySQLConfig) dsn() string {
 		port = 3306
 	}
 	// parseTime + UTC so DATETIME columns scan into time.Time; utf8mb4 for full
-	// Unicode (names, comments…).
-	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&loc=UTC&charset=utf8mb4",
+	// Unicode (names, comments…). timeout bounds the TCP dial so an unreachable
+	// or wrong-port server fails fast with a real error instead of hanging
+	// startup (which would surface only as "db not initialized"); read/write
+	// timeouts cap a stuck statement (generous, so normal migrations/imports
+	// never trip them).
+	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&loc=UTC&charset=utf8mb4&timeout=10s&readTimeout=120s&writeTimeout=120s",
 		c.User, c.Password, c.Host, port, c.Database)
 }
 
@@ -50,8 +54,12 @@ func validDBIdent(s string) bool {
 
 // PingMySQL verifies a shared-database connection and creates the logbook
 // database if it doesn't exist yet. It connects at server level first (no
-// database selected) so a not-yet-created DB isn't an error, then runs
-// CREATE DATABASE IF NOT EXISTS. Backs the settings "Test connection" button.
+// database selected), tries CREATE DATABASE IF NOT EXISTS, then confirms the
+// database is actually usable. A restricted user (common on shared hosting)
+// may lack the CREATE DATABASE privilege but still have full rights on a
+// pre-created database — so a denied CREATE is not fatal as long as the
+// database already exists and we can connect to it. Backs the "Test
+// connection" button.
 func PingMySQL(c MySQLConfig) error {
 	if strings.TrimSpace(c.Host) == "" {
 		return fmt.Errorf("host is required")
@@ -63,17 +71,37 @@ func PingMySQL(c MySQLConfig) error {
 		return fmt.Errorf("open mysql: %w", err)
 	}
 	defer conn.Close()
-	conn.SetConnMaxLifetime(5 * time.Second)
+	conn.SetConnMaxLifetime(15 * time.Second)
 	if err := conn.Ping(); err != nil {
 		return fmt.Errorf("connect to %s:%d: %w", c.Host, c.Port, err)
 	}
-	if name := strings.TrimSpace(c.Database); name != "" {
-		if !validDBIdent(name) {
-			return fmt.Errorf("invalid database name %q (letters, digits, underscore only)", name)
-		}
-		if _, err := conn.Exec("CREATE DATABASE IF NOT EXISTS `" + name + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil {
+	name := strings.TrimSpace(c.Database)
+	if name == "" {
+		return nil
+	}
+	if !validDBIdent(name) {
+		return fmt.Errorf("invalid database name %q (letters, digits, underscore only)", name)
+	}
+	createErr := error(nil)
+	if _, err := conn.Exec("CREATE DATABASE IF NOT EXISTS `" + name + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil {
+		if !isAccessDenied(err) {
 			return fmt.Errorf("create database %q: %w", name, err)
 		}
+		createErr = err // remember it in case the DB also turns out to be unreachable
+	}
+	// Confirm the database is usable (it may have been pre-created by an admin
+	// even though this user can't CREATE one).
+	dbConn, err := sql.Open("mysql", c.dsn())
+	if err != nil {
+		return fmt.Errorf("open mysql db: %w", err)
+	}
+	defer dbConn.Close()
+	dbConn.SetConnMaxLifetime(15 * time.Second)
+	if err := dbConn.Ping(); err != nil {
+		if createErr != nil {
+			return fmt.Errorf("database %q does not exist and user %q cannot create it — ask your MySQL admin to create the database and grant access (%v)", name, c.User, createErr)
+		}
+		return fmt.Errorf("connect to database %q: %w", name, err)
 	}
 	return nil
 }
@@ -81,6 +109,13 @@ func PingMySQL(c MySQLConfig) error {
 //go:embed migrations/*.sql
 var migrationsFS embed.FS
 
+// schemaMigrationsDDL tracks which migrations have run. Authored in SQLite
+// dialect; run through the dialect translator for MySQL.
+const schemaMigrationsDDL = `CREATE TABLE IF NOT EXISTS schema_migrations (
+        name TEXT PRIMARY KEY,
+        applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    )`
+
 // Dialect names the backend opened at startup. Repos consult it for the few
 // places where SQLite and MySQL SQL genuinely differ (upserts, JSON extraction).
 // Timestamps are generated in Go (NowISO) for both, so most queries are shared.
@@ -138,13 +173,15 @@ func Open(path string) (*sql.DB, error) {
 // (no external dependency). translate, when non-nil, rewrites each statement
 // for a non-SQLite backend (see mysqlDDL); nil means run the SQLite DDL as-is.
 func migrate(conn *sql.DB, translate func(string) string) error {
+	// A non-nil translator means this is the MySQL connection (use the
+	// per-statement, FK-aware path); nil means a SQLite connection. This is
+	// determined by the caller's argument, NOT the global Dialect, so the
+	// in-memory SQLite used to build the MySQL baseline still migrates as SQLite.
+	mysqlPath := translate != nil
 	if translate == nil {
 		translate = func(s string) string { return s }
 	}
-	if _, err := conn.Exec(translate(`CREATE TABLE IF NOT EXISTS schema_migrations (
-        name TEXT PRIMARY KEY,
-        applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
-    )`)); err != nil {
+	if _, err := conn.Exec(translate(schemaMigrationsDDL)); err != nil {
 		return fmt.Errorf("create schema_migrations: %w", err)
 	}
 
@@ -182,7 +219,7 @@ func migrate(conn *sql.DB, translate func(string) string) error {
 		// "duplicate column". Instead apply statement-by-statement and tolerate
 		// "already exists" errors, making migrations idempotent (and self-healing
 		// for a previously half-applied database).
-		if IsMySQL() {
+		if mysqlPath {
 			if err := applyMySQLMigration(conn, sqlText); err != nil {
 				return fmt.Errorf("apply migration %s: %w", name, err)
 			}

internal/db/migrations/0022_profile_db.sql

@@ -0,0 +1,6 @@
+-- Per-profile logbook database. Each profile can target its own logbook:
+-- the local SQLite file, or a specific shared MySQL database. Switching the
+-- active profile switches the logbook accordingly. Stored as a small JSON
+-- document {backend, host, port, user, password, database}; empty = inherit
+-- the default (local SQLite).
+ALTER TABLE station_profiles ADD COLUMN db_config TEXT NOT NULL DEFAULT '';

internal/db/mysql.go

@@ -37,6 +37,11 @@ var (
 	reBareInteger     = regexp.MustCompile(`\bINTEGER\b`)
 	reColText         = regexp.MustCompile(`(\w+)\s+TEXT\b`)
 	reKeyColumn       = regexp.MustCompile(`(?m)^(\s*)key\b`)
+	// SQLite quotes identifiers with double quotes — e.g. a table renamed by
+	// ALTER … RENAME TO ends up as CREATE TABLE "name" in sqlite_master. MySQL
+	// uses backticks. Our schema has no double-quoted string literals, so it's
+	// safe to convert every "ident" to `ident`.
+	reDoubleQuoteIdent = regexp.MustCompile(`"([^"\n]*)"`)
 )
 
 // MySQL's InnoDB row-size limit is 65535 bytes (excluding off-page TEXT/BLOB).
@@ -71,6 +76,9 @@ var longTextColumns = map[string]string{
 // file) into the MySQL dialect. See the package note above for the rules.
 func mysqlDDL(stmt string) string {
 	s := stmt
+	// SQLite double-quoted identifiers → MySQL backticks (renamed tables in the
+	// baseline dump, etc.).
+	s = reDoubleQuoteIdent.ReplaceAllString(s, "`$1`")
 	// MySQL has no IF NOT EXISTS for CREATE INDEX (migrations run once anyway).
 	s = strings.ReplaceAll(s, "CREATE INDEX IF NOT EXISTS", "CREATE INDEX")
 	// Auto-increment primary keys. Both the AUTOINCREMENT form and the bare
@@ -144,7 +152,11 @@ func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
 	conn.SetMaxOpenConns(50)
 	conn.SetMaxIdleConns(10)
 	conn.SetConnMaxIdleTime(90 * time.Second)
-	conn.SetConnMaxLifetime(5 * time.Minute)
+	// No max lifetime: a slow server's first migration can run for minutes on a
+	// single connection, and reaping it mid-migration drops the selected database
+	// (surfacing as "Unknown database"). Idle connections are still recycled
+	// after 90s, and the driver retries stale pooled connections.
+	conn.SetConnMaxLifetime(0)
 	if err := conn.Ping(); err != nil {
 		_ = conn.Close()
 		return nil, fmt.Errorf("connect to %s: %w", name, err)
@@ -152,7 +164,22 @@ func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
 	// Set the dialect before migrating so the runner takes the MySQL path
 	// (per-statement, idempotent) rather than the SQLite transaction path.
 	Dialect = "mysql"
-	if err := migrate(conn, mysqlDDL); err != nil {
+	fresh, err := isFreshMySQL(conn)
+	if err != nil {
+		_ = conn.Close()
+		Dialect = "sqlite"
+		return nil, fmt.Errorf("inspect database: %w", err)
+	}
+	if fresh {
+		// Empty database: build the whole final schema in one pass (one CREATE
+		// per table, no ALTERs) — far faster than replaying 21 migrations,
+		// especially on a server with slow DDL.
+		err = applyMySQLBaseline(conn)
+	} else {
+		// Existing database: apply only the migrations it's missing.
+		err = migrate(conn, mysqlDDL)
+	}
+	if err != nil {
 		_ = conn.Close()
 		Dialect = "sqlite" // migration failed; we'll fall back to SQLite
 		return nil, err
@@ -160,6 +187,116 @@ func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
 	return conn, nil
 }
 
+// isFreshMySQL reports whether the database has no OpsLog schema yet (no qso
+// table), so the baseline fast-path applies. A partially-migrated database is
+// NOT fresh and goes through the incremental migrator.
+func isFreshMySQL(conn *sql.DB) (bool, error) {
+	var name string
+	err := conn.QueryRow("SHOW TABLES LIKE 'qso'").Scan(&name)
+	if err == sql.ErrNoRows {
+		return true, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// applyMySQLBaseline creates the entire current schema on a fresh MySQL database
+// in a single pass, then records every migration as already applied. The schema
+// is derived from the migrations themselves (replayed on a throwaway in-memory
+// SQLite, whose sqlite_master holds each table's FINAL CREATE statement with all
+// ALTER-added columns folded in) and translated to MySQL — so there's no second
+// schema to maintain and the two backends can't drift. Future migrations apply
+// incrementally on top.
+func applyMySQLBaseline(conn *sql.DB) error {
+	mem, err := sql.Open("sqlite", "file:opslog_baseline?mode=memory&cache=shared")
+	if err != nil {
+		return fmt.Errorf("open baseline sqlite: %w", err)
+	}
+	defer mem.Close()
+	if err := migrate(mem, nil); err != nil {
+		return fmt.Errorf("build baseline schema: %w", err)
+	}
+
+	var migNames []string
+	mrows, err := mem.Query("SELECT name FROM schema_migrations ORDER BY name")
+	if err != nil {
+		return fmt.Errorf("read baseline migrations: %w", err)
+	}
+	for mrows.Next() {
+		var n string
+		if err := mrows.Scan(&n); err != nil {
+			mrows.Close()
+			return err
+		}
+		migNames = append(migNames, n)
+	}
+	mrows.Close()
+
+	var tables, indexes []string
+	srows, err := mem.Query("SELECT type, sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' AND name != 'schema_migrations'")
+	if err != nil {
+		return fmt.Errorf("read baseline schema: %w", err)
+	}
+	for srows.Next() {
+		var typ, s string
+		if err := srows.Scan(&typ, &s); err != nil {
+			srows.Close()
+			return err
+		}
+		switch typ {
+		case "table":
+			tables = append(tables, s)
+		case "index":
+			indexes = append(indexes, s)
+		}
+	}
+	srows.Close()
+
+	// Apply on one connection with FK checks off so table order doesn't matter.
+	ctx := context.Background()
+	c, err := conn.Conn(ctx)
+	if err != nil {
+		return fmt.Errorf("acquire conn: %w", err)
+	}
+	defer func() {
+		_, _ = c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=1")
+		_ = c.Close()
+	}()
+	if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil {
+		return fmt.Errorf("disable fk checks: %w", err)
+	}
+	if _, err := c.ExecContext(ctx, mysqlDDL(schemaMigrationsDDL)); err != nil {
+		return fmt.Errorf("create schema_migrations: %w", err)
+	}
+	for _, tb := range tables {
+		if _, err := c.ExecContext(ctx, mysqlDDL(tb)); err != nil {
+			if isIgnorableDDLError(err) {
+				continue
+			}
+			return fmt.Errorf("baseline table: %w", err)
+		}
+	}
+	for _, ix := range indexes {
+		if _, err := c.ExecContext(ctx, mysqlDDL(ix)); err != nil {
+			if isIgnorableDDLError(err) {
+				continue
+			}
+			return fmt.Errorf("baseline index: %w", err)
+		}
+	}
+	for _, n := range migNames {
+		if _, err := conn.Exec("INSERT INTO schema_migrations(name) VALUES(?)", n); err != nil {
+			if isIgnorableDDLError(err) {
+				continue
+			}
+			return fmt.Errorf("record migration %s: %w", n, err)
+		}
+	}
+	return nil
+}
+
 // applyMySQLMigration executes a translated migration one statement at a time.
 // MySQL has no multi-statement Exec without a special flag and auto-commits DDL,
 // so running statements individually (and tolerating "already exists" errors)
@@ -182,9 +319,19 @@ func applyMySQLMigration(conn *sql.DB, sqlText string) error {
 	if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil {
 		return fmt.Errorf("disable fk checks: %w", err)
 	}
-	for _, stmt := range splitStatements(sqlText) {
+	for _, stmt := range coalesceAddColumns(splitStatements(sqlText)) {
 		if _, err := c.ExecContext(ctx, stmt); err != nil {
 			if isIgnorableDDLError(err) {
+				// A coalesced multi-column ALTER fails wholesale if even one
+				// column already exists (partial prior apply). Re-run it column
+				// by column so the missing ones still get added.
+				if cols := decomposeAddColumns(stmt); len(cols) > 1 {
+					for _, one := range cols {
+						if _, e := c.ExecContext(ctx, one); e != nil && !isIgnorableDDLError(e) {
+							return fmt.Errorf("statement %q: %w", firstLine(one), e)
+						}
+					}
+				}
 				continue
 			}
 			return fmt.Errorf("statement %q: %w", firstLine(stmt), err)
@@ -193,13 +340,65 @@ func applyMySQLMigration(conn *sql.DB, sqlText string) error {
 	return nil
 }
 
+var reAddColumn = regexp.MustCompile(`(?is)^\s*ALTER\s+TABLE\s+(\S+)\s+ADD\s+COLUMN\s+(.+)$`)
+
+// coalesceAddColumns merges consecutive "ALTER TABLE t ADD COLUMN …" statements
+// on the same table into a single multi-column ALTER. SQLite needs one ALTER per
+// column, but each is a full table operation in MySQL — and on a slow server
+// that's seconds apiece, turning a ~50-column migration into minutes (long
+// enough to trip connection lifetimes and stall startup). One combined ALTER is
+// a single table rebuild instead of fifty.
+func coalesceAddColumns(stmts []string) []string {
+	out := make([]string, 0, len(stmts))
+	for i := 0; i < len(stmts); {
+		m := reAddColumn.FindStringSubmatch(stmts[i])
+		if m == nil {
+			out = append(out, stmts[i])
+			i++
+			continue
+		}
+		table := m[1]
+		cols := []string{strings.TrimSpace(m[2])}
+		j := i + 1
+		for j < len(stmts) {
+			m2 := reAddColumn.FindStringSubmatch(stmts[j])
+			if m2 == nil || m2[1] != table {
+				break
+			}
+			cols = append(cols, strings.TrimSpace(m2[2]))
+			j++
+		}
+		out = append(out, "ALTER TABLE "+table+" ADD COLUMN "+strings.Join(cols, ", ADD COLUMN "))
+		i = j
+	}
+	return out
+}
+
+// decomposeAddColumns splits a (possibly coalesced) ADD COLUMN ALTER back into
+// one ALTER per column, for the idempotent fallback path.
+func decomposeAddColumns(stmt string) []string {
+	m := reAddColumn.FindStringSubmatch(stmt)
+	if m == nil {
+		return nil
+	}
+	table := m[1]
+	var out []string
+	for _, col := range strings.Split(m[2], ", ADD COLUMN ") {
+		out = append(out, "ALTER TABLE "+table+" ADD COLUMN "+strings.TrimSpace(col))
+	}
+	return out
+}
+
 // splitStatements breaks a migration file into individual SQL statements,
-// dropping full-line comments and blank fragments. Our migrations never embed
-// a ';' inside a string literal, so a simple split is safe.
+// dropping comments (full-line and inline "-- …") and blank fragments. Our
+// migrations never embed a ';' or '--' inside a string literal, so this is safe.
 func splitStatements(sqlText string) []string {
 	var b strings.Builder
 	for _, line := range strings.Split(sqlText, "\n") {
-		if strings.HasPrefix(strings.TrimSpace(line), "--") {
+		if i := strings.Index(line, "--"); i >= 0 {
+			line = line[:i] // strip inline / full-line comment
+		}
+		if strings.TrimSpace(line) == "" {
 			continue
 		}
 		b.WriteString(line)
@@ -222,6 +421,24 @@ func firstLine(s string) string {
 	return s
 }
 
+// isAccessDenied reports whether a MySQL error is a privilege denial — e.g. a
+// restricted user trying to CREATE DATABASE. Such users can still operate on a
+// pre-existing database they've been granted, so the caller treats this as
+// non-fatal and verifies database access separately.
+func isAccessDenied(err error) bool {
+	var me *mysqldriver.MySQLError
+	if !errors.As(err, &me) {
+		return false
+	}
+	switch me.Number {
+	case 1044, // ER_DBACCESS_DENIED_ERROR — access denied to database
+		1045, // ER_ACCESS_DENIED_ERROR — access denied for user
+		1142: // ER_TABLEACCESS_DENIED_ERROR — command denied to user
+		return true
+	}
+	return false
+}
+
 // isIgnorableDDLError reports whether a MySQL error means the object the
 // statement creates/drops is already in the intended state — safe to skip when
 // re-applying a forward-only migration.