OpsLog/internal/db/mysql.go

package db

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"regexp"
	"strings"
	"time"

	mysqldriver "github.com/go-sql-driver/mysql"
)

// ── SQLite → MySQL schema translation ──────────────────────────────────
//
// OpsLog's migrations are authored in SQLite dialect (the default backend).
// For the shared-MySQL backend we translate that DDL on the fly rather than
// maintaining a second set of migration files, so the two backends can never
// drift apart. The translation is deliberately narrow — it only rewrites the
// SQLite-isms that actually appear in our migrations:
//
//   - INTEGER PRIMARY KEY [AUTOINCREMENT]  → BIGINT AUTO_INCREMENT PRIMARY KEY
//   - bare INTEGER                          → BIGINT (so FK child/parent types match)
//   - DEFAULT (strftime(... 'now'))         → DEFAULT '' (timestamps come from Go)
//   - TEXT                                  → VARCHAR(255), except a few free-text
//                                             columns that stay TEXT/LONGTEXT
//   - reserved column `key`                 → backticked
//   - CREATE INDEX IF NOT EXISTS            → CREATE INDEX (MySQL has no IF NOT EXISTS)
//
// REAL is left as-is (MySQL accepts it as an alias for DOUBLE). CREATE/DROP
// TABLE IF [NOT] EXISTS, RENAME TO, INSERT…SELECT, CHECK(...) and foreign keys
// are all valid MySQL as written.

var (
	reStrftimeDefault = regexp.MustCompile(`\(strftime\('%Y-%m-%dT%H:%M:%fZ',\s*'now'\)\)`)
	reBareInteger     = regexp.MustCompile(`\bINTEGER\b`)
	// The column name may be bare or quoted. The SQLite baseline dump quotes its
	// identifiers, and those double quotes are already BACKTICKS by the time this
	// runs — so a pattern matching only a bare name left every column in the
	// baseline as TEXT. That produced both "BLOB/TEXT column 'state' used in key
	// specification without a key length" and "column 'op_name' can't have a
	// default value" on a MySQL logbook, neither of which names the real cause.
	reColText         = regexp.MustCompile("(`?)(\\w+)(`?)\\s+TEXT\\b")
	reKeyColumn       = regexp.MustCompile(`(?m)^(\s*)key\b`)
	// SQLite quotes identifiers with double quotes — e.g. a table renamed by
	// ALTER … RENAME TO ends up as CREATE TABLE "name" in sqlite_master. MySQL
	// uses backticks. Our schema has no double-quoted string literals, so it's
	// safe to convert every "ident" to `ident`.
	reDoubleQuoteIdent = regexp.MustCompile(`"([^"\n]*)"`)
)

// MySQL's InnoDB row-size limit is 65535 bytes (excluding off-page TEXT/BLOB).
// The qso table has ~150 columns, so making them all VARCHAR(255) (1020 bytes
// each in utf8mb4) overflows the row. We therefore emit TEXT (stored off-page,
// ~20 bytes in-row) for the bulk of columns, and VARCHAR only where a column
// actually needs in-row storage: it's indexed, a primary key, or carries a
// DEFAULT (TEXT can't hold a literal default in MySQL).

// varcharColumns must be VARCHAR because they're indexed or part of a primary
// key declared on a separate line (so the DEFAULT/PRIMARY-KEY line heuristic
// below can't catch them). MySQL can't index a TEXT column without a prefix
// length. Keyed by column name (a name may repeat across tables — harmless).
var varcharColumns = map[string]bool{
	// qso indexes (0001, 0003, 0019)
	"callsign": true, "qso_date": true, "band": true, "mode": true,
	"grid": true, "station_callsign": true, "state": true, "contest_id": true,
	"sat_name": true, "prop_mode": true, "sig": true, "wwff_ref": true, "skcc": true,
	// integrations_udp index (0011)
	"direction": true,
	// award_references composite primary key (0017)
	"award_code": true, "ref_code": true,
}

// longTextColumns override the default TEXT with a bigger type where the
// content can be large (a full QSL-template document).
var longTextColumns = map[string]string{
	"json": "LONGTEXT",
}

// mysqlDDL rewrites a SQLite DDL/DML statement (or multi-statement migration
// file) into the MySQL dialect. See the package note above for the rules.
func mysqlDDL(stmt string) string {
	s := stmt
	// SQLite double-quoted identifiers → MySQL backticks (renamed tables in the
	// baseline dump, etc.).
	s = reDoubleQuoteIdent.ReplaceAllString(s, "`$1`")
	// MySQL has no IF NOT EXISTS for CREATE INDEX (migrations run once anyway).
	s = strings.ReplaceAll(s, "CREATE INDEX IF NOT EXISTS", "CREATE INDEX")
	// Auto-increment primary keys. Both the AUTOINCREMENT form and the bare
	// "INTEGER PRIMARY KEY" (SQLite rowid alias) must become AUTO_INCREMENT so
	// inserts that omit the id still get one.
	s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY AUTOINCREMENT", "BIGINT AUTO_INCREMENT PRIMARY KEY")
	s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY", "BIGINT AUTO_INCREMENT PRIMARY KEY")
	// Timestamp defaults: Go supplies ISO strings, so drop the SQLite function
	// default (TEXT/VARCHAR can't carry it in MySQL anyway).
	s = reStrftimeDefault.ReplaceAllString(s, "''")
	// Remaining INTEGER columns → BIGINT so foreign-key parent/child types match.
	s = reBareInteger.ReplaceAllString(s, "BIGINT")
	// TEXT columns → TEXT (off-page) by default, VARCHAR(255) only when indexed,
	// a primary key, or default-bearing — keeping the row under MySQL's limit.
	s = translateTextColumns(s)
	// `key` is a MySQL reserved word — backtick the settings column declaration.
	s = reKeyColumn.ReplaceAllString(s, "$1`key`")
	return s
}

// translateTextColumns rewrites each "<col> TEXT" column declaration to the
// right MySQL type, deciding per line so it can see whether the line carries a
// DEFAULT or an inline PRIMARY KEY. See varcharColumns / longTextColumns.
func translateTextColumns(s string) string {
	lines := strings.Split(s, "\n")
	for i, line := range lines {
		m := reColText.FindStringSubmatchIndex(line)
		if m == nil {
			continue
		}
		// Groups: 1 = opening backtick (or empty), 2 = the name, 3 = closing one.
		openQ, col, closeQ := line[m[2]:m[3]], line[m[4]:m[5]], line[m[6]:m[7]]
		var typ string
		switch {
		case longTextColumns[col] != "":
			typ = longTextColumns[col]
		case varcharColumns[col],
			strings.Contains(line, "DEFAULT"),
			strings.Contains(line, "PRIMARY KEY"):
			typ = "VARCHAR(255)"
		default:
			typ = "TEXT"
		}
		lines[i] = line[:m[0]] + openQ + col + closeQ + " " + typ + line[m[1]:]
	}
	return strings.Join(lines, "\n")
}

// OpenMySQL opens the shared MySQL logbook, creating the database if needed,
// then applies the (translated) embedded migrations. multiStatements is enabled
// so multi-statement migration files run in a single Exec.
// tuneMySQLPool sizes the connection pool for a SHARED server. Two opposing needs:
//   - A big per-instance pool (we shipped 50) let a handful of ops exhaust the
//     server's global max_connections — "Error 1040: Too many connections".
//   - Too SMALL a pool starves this instance's OWN concurrent UI queries — the live
//     multi-op sync (revision poll every 2s + a 3-query grid refresh), live_status,
//     chat, stats, cluster. Under real multi-op load, 8 and even 16 stalled: hot
//     paths (WorkedBefore fires ~10 sequential round-trips; every poll uses the
//     app-lifetime ctx with no per-query timeout) hold a connection for the whole
//     chain of remote-MySQL latency, so a handful of concurrent UI bursts drained
//     the pool and "after a while nothing updated" — grid, chat, live_status froze.
//
// 40 fits the actual deployment (max 5 ops on a server with max_connections=300 →
// 40*5 = 200, leaving ~100 for phpMyAdmin/server overhead). This is the per-INSTANCE
// pool, so keep max_connections >= pool*ops + headroom.
// Brisk idle reaping returns slots to the server; no max lifetime so a slow first
// migration on one connection isn't reaped mid-run (drops the selected database).
func tuneMySQLPool(conn *sql.DB) {
	conn.SetMaxOpenConns(40)
	conn.SetMaxIdleConns(8)
	conn.SetConnMaxIdleTime(30 * time.Second)
	conn.SetConnMaxLifetime(0)
}

func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
	if strings.TrimSpace(c.Host) == "" {
		return nil, fmt.Errorf("host is required")
	}
	name := strings.TrimSpace(c.Database)
	if !validDBIdent(name) {
		return nil, fmt.Errorf("invalid database name %q (letters, digits, underscore only)", name)
	}
	conn, err := sql.Open("mysql", c.dsn())
	if err != nil {
		return nil, fmt.Errorf("open mysql: %w", err)
	}
	tuneMySQLPool(conn)
	// Connect DIRECTLY to the database first. Only if that fails — the DB doesn't
	// exist yet (first-time setup) or the server is unreachable — do we pay for the
	// server-level connect + CREATE DATABASE (two more handshakes). On a normal
	// startup against an existing DB this halves the connection round-trips, which
	// matters a lot on a high-latency remote MySQL (each handshake is several RTTs).
	if err := conn.Ping(); err != nil {
		_ = conn.Close()
		if perr := PingMySQL(c); perr != nil {
			return nil, perr // creates the DB (or returns a clear "cannot create" error)
		}
		conn, err = sql.Open("mysql", c.dsn())
		if err != nil {
			return nil, fmt.Errorf("open mysql: %w", err)
		}
		tuneMySQLPool(conn)
		if err := conn.Ping(); err != nil {
			_ = conn.Close()
			return nil, fmt.Errorf("connect to %s: %w", name, err)
		}
	}
	// Set the dialect before migrating so the runner takes the MySQL path
	// (per-statement, idempotent) rather than the SQLite transaction path.
	Dialect = "mysql"
	fresh, err := isFreshMySQL(conn)
	if err != nil {
		_ = conn.Close()
		Dialect = "sqlite"
		return nil, fmt.Errorf("inspect database: %w", err)
	}
	if fresh {
		// Empty database: build the whole final schema in one pass (one CREATE
		// per table, no ALTERs) — far faster than replaying 21 migrations,
		// especially on a server with slow DDL.
		err = applyMySQLBaseline(conn)
	} else {
		// Existing database: apply only the migrations it's missing.
		err = migrate(conn, mysqlDDL, "", "mysql:"+name)
	}
	if err != nil {
		_ = conn.Close()
		Dialect = "sqlite" // migration failed; we'll fall back to SQLite
		return nil, err
	}
	// Guarantee the schema can store non-Latin text (Cyrillic, Polish ł, …).
	// Best-effort: never block access to the logbook over the charset probe.
	if err := ensureMySQLUTF8MB4(conn, name); err != nil {
		fmt.Printf("OpsLog: utf8mb4 conversion: %v\n", err)
	}
	return conn, nil
}

// ensureMySQLUTF8MB4 makes the database default and every OpsLog table utf8mb4,
// converting any that isn't. It repairs databases pre-created as latin1 or
// utf8mb3 by a hosting panel (cPanel/o2switch default), where OpsLog's
// "CREATE DATABASE IF NOT EXISTS … utf8mb4" was a no-op and the tables inherited
// a charset that can't store non-Latin callsigns/names/QTHs — the MySQL 1366
// "Incorrect string value: '\xD0\xAF'" (Cyrillic Я) seen updating a QSO from QRZ.
//
// Idempotent and cheap on a healthy database: one information_schema probe finds
// the tables with a non-utf8mb4 text column, and only those are converted (none
// on an already-correct DB). Conversion is safe for the wide qso table because
// the bulk of its columns are off-page TEXT (see the row-size note above); only
// the ~20 indexed VARCHARs sit in-row, well under InnoDB's 65535-byte limit even
// at 4 bytes/char.
func ensureMySQLUTF8MB4(conn *sql.DB, dbName string) error {
	// Fix the database default so any FUTURE table is utf8mb4 too.
	_, _ = conn.Exec("ALTER DATABASE `" + dbName + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")

	rows, err := conn.Query(`
		SELECT DISTINCT TABLE_NAME
		FROM information_schema.COLUMNS
		WHERE TABLE_SCHEMA = ?
		  AND CHARACTER_SET_NAME IS NOT NULL
		  AND CHARACTER_SET_NAME <> 'utf8mb4'`, dbName)
	if err != nil {
		return fmt.Errorf("probe charsets: %w", err)
	}
	var tables []string
	for rows.Next() {
		var t string
		if err := rows.Scan(&t); err == nil && validDBIdent(t) {
			tables = append(tables, t)
		}
	}
	rows.Close()

	var firstErr error
	for _, t := range tables {
		if _, err := conn.Exec("ALTER TABLE `" + t + "` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil && firstErr == nil {
			firstErr = fmt.Errorf("convert %s: %w", t, err)
		}
	}
	return firstErr
}

// isFreshMySQL reports whether the database has no OpsLog schema yet (no qso
// table), so the baseline fast-path applies. A partially-migrated database is
// NOT fresh and goes through the incremental migrator.
func isFreshMySQL(conn *sql.DB) (bool, error) {
	var name string
	err := conn.QueryRow("SHOW TABLES LIKE 'qso'").Scan(&name)
	if err == sql.ErrNoRows {
		return true, nil
	}
	if err != nil {
		return false, err
	}
	return false, nil
}

// applyMySQLBaseline creates the entire current schema on a fresh MySQL database
// in a single pass, then records every migration as already applied. The schema
// is derived from the migrations themselves (replayed on a throwaway in-memory
// SQLite, whose sqlite_master holds each table's FINAL CREATE statement with all
// ALTER-added columns folded in) and translated to MySQL — so there's no second
// schema to maintain and the two backends can't drift. Future migrations apply
// incrementally on top.
func applyMySQLBaseline(conn *sql.DB) error {
	mem, err := sql.Open("sqlite", "file:opslog_baseline?mode=memory&cache=shared")
	if err != nil {
		return fmt.Errorf("open baseline sqlite: %w", err)
	}
	defer mem.Close()
	// In-memory SQLite, used only to derive the final schema for a FRESH MySQL
	// database. Labelled so its (fast) migration lines are not mistaken for a
	// real database being migrated — in one operator's log this pass sat between
	// two slow MySQL runs and looked like a third database.
	if err := migrate(mem, nil, "", "baseline:memory"); err != nil {
		return fmt.Errorf("build baseline schema: %w", err)
	}

	var migNames []string
	mrows, err := mem.Query("SELECT name FROM schema_migrations ORDER BY name")
	if err != nil {
		return fmt.Errorf("read baseline migrations: %w", err)
	}
	for mrows.Next() {
		var n string
		if err := mrows.Scan(&n); err != nil {
			mrows.Close()
			return err
		}
		migNames = append(migNames, n)
	}
	mrows.Close()

	var tables, indexes []string
	srows, err := mem.Query("SELECT type, sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' AND name != 'schema_migrations'")
	if err != nil {
		return fmt.Errorf("read baseline schema: %w", err)
	}
	for srows.Next() {
		var typ, s string
		if err := srows.Scan(&typ, &s); err != nil {
			srows.Close()
			return err
		}
		switch typ {
		case "table":
			tables = append(tables, s)
		case "index":
			indexes = append(indexes, s)
		}
	}
	srows.Close()

	// Apply on one connection with FK checks off so table order doesn't matter.
	ctx := context.Background()
	c, err := conn.Conn(ctx)
	if err != nil {
		return fmt.Errorf("acquire conn: %w", err)
	}
	defer func() {
		_, _ = c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=1")
		_ = c.Close()
	}()
	if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil {
		return fmt.Errorf("disable fk checks: %w", err)
	}
	if _, err := c.ExecContext(ctx, mysqlDDL(schemaMigrationsDDL)); err != nil {
		return fmt.Errorf("create schema_migrations: %w", err)
	}
	for _, tb := range tables {
		if _, err := c.ExecContext(ctx, mysqlDDL(tb)); err != nil {
			if isIgnorableDDLError(err) {
				continue
			}
			return fmt.Errorf("baseline table: %w", err)
		}
	}
	for _, ix := range indexes {
		if _, err := c.ExecContext(ctx, mysqlDDL(ix)); err != nil {
			if isIgnorableDDLError(err) {
				continue
			}
			return fmt.Errorf("baseline index: %w", err)
		}
	}
	for _, n := range migNames {
		if _, err := conn.Exec("INSERT INTO schema_migrations(name) VALUES(?)", n); err != nil {
			if isIgnorableDDLError(err) {
				continue
			}
			return fmt.Errorf("record migration %s: %w", n, err)
		}
	}
	return nil
}

// applyMySQLMigration executes a translated migration one statement at a time.
// MySQL has no multi-statement Exec without a special flag and auto-commits DDL,
// so running statements individually (and tolerating "already exists" errors)
// keeps migrations idempotent and lets a half-applied database self-heal.
//
// It runs on a single dedicated connection with FOREIGN_KEY_CHECKS=0, because a
// few migrations recreate tables by DROP/RENAME in an order that trips MySQL's
// foreign-key enforcement (SQLite is laxer). The flag is reset to 1 before the
// connection returns to the pool, so runtime queries keep FK enforcement.
func applyMySQLMigration(conn *sql.DB, sqlText string) error {
	ctx := context.Background()
	c, err := conn.Conn(ctx)
	if err != nil {
		return fmt.Errorf("acquire conn: %w", err)
	}
	defer func() {
		_, _ = c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=1")
		_ = c.Close()
	}()
	if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil {
		return fmt.Errorf("disable fk checks: %w", err)
	}
	for _, stmt := range coalesceAddColumns(splitStatements(sqlText)) {
		if _, err := c.ExecContext(ctx, stmt); err != nil {
			if isIgnorableDDLError(err) {
				// A coalesced multi-column ALTER fails wholesale if even one
				// column already exists (partial prior apply). Re-run it column
				// by column so the missing ones still get added.
				if cols := decomposeAddColumns(stmt); len(cols) > 1 {
					for _, one := range cols {
						if _, e := c.ExecContext(ctx, one); e != nil && !isIgnorableDDLError(e) {
							return fmt.Errorf("statement %q: %w", firstLine(one), e)
						}
					}
				}
				continue
			}
			return fmt.Errorf("statement %q: %w", firstLine(stmt), err)
		}
	}
	return nil
}

var reAddColumn = regexp.MustCompile(`(?is)^\s*ALTER\s+TABLE\s+(\S+)\s+ADD\s+COLUMN\s+(.+)$`)

// coalesceAddColumns merges consecutive "ALTER TABLE t ADD COLUMN …" statements
// on the same table into a single multi-column ALTER. SQLite needs one ALTER per
// column, but each is a full table operation in MySQL — and on a slow server
// that's seconds apiece, turning a ~50-column migration into minutes (long
// enough to trip connection lifetimes and stall startup). One combined ALTER is
// a single table rebuild instead of fifty.
func coalesceAddColumns(stmts []string) []string {
	out := make([]string, 0, len(stmts))
	for i := 0; i < len(stmts); {
		m := reAddColumn.FindStringSubmatch(stmts[i])
		if m == nil {
			out = append(out, stmts[i])
			i++
			continue
		}
		table := m[1]
		cols := []string{strings.TrimSpace(m[2])}
		j := i + 1
		for j < len(stmts) {
			m2 := reAddColumn.FindStringSubmatch(stmts[j])
			if m2 == nil || m2[1] != table {
				break
			}
			cols = append(cols, strings.TrimSpace(m2[2]))
			j++
		}
		out = append(out, "ALTER TABLE "+table+" ADD COLUMN "+strings.Join(cols, ", ADD COLUMN "))
		i = j
	}
	return out
}

// decomposeAddColumns splits a (possibly coalesced) ADD COLUMN ALTER back into
// one ALTER per column, for the idempotent fallback path.
func decomposeAddColumns(stmt string) []string {
	m := reAddColumn.FindStringSubmatch(stmt)
	if m == nil {
		return nil
	}
	table := m[1]
	var out []string
	for _, col := range strings.Split(m[2], ", ADD COLUMN ") {
		out = append(out, "ALTER TABLE "+table+" ADD COLUMN "+strings.TrimSpace(col))
	}
	return out
}

// splitStatements breaks a migration file into individual SQL statements,
// dropping comments (full-line and inline "-- …") and blank fragments. Our
// migrations never embed a ';' or '--' inside a string literal, so this is safe.
func splitStatements(sqlText string) []string {
	var b strings.Builder
	for _, line := range strings.Split(sqlText, "\n") {
		if i := strings.Index(line, "--"); i >= 0 {
			line = line[:i] // strip inline / full-line comment
		}
		if strings.TrimSpace(line) == "" {
			continue
		}
		b.WriteString(line)
		b.WriteByte('\n')
	}
	var out []string
	for _, part := range strings.Split(b.String(), ";") {
		if strings.TrimSpace(part) != "" {
			out = append(out, part)
		}
	}
	return out
}

func firstLine(s string) string {
	s = strings.TrimSpace(s)
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		return s[:i]
	}
	return s
}

// isAccessDenied reports whether a MySQL error is a privilege denial — e.g. a
// restricted user trying to CREATE DATABASE. Such users can still operate on a
// pre-existing database they've been granted, so the caller treats this as
// non-fatal and verifies database access separately.
func isAccessDenied(err error) bool {
	var me *mysqldriver.MySQLError
	if !errors.As(err, &me) {
		return false
	}
	switch me.Number {
	case 1044, // ER_DBACCESS_DENIED_ERROR — access denied to database
		1045, // ER_ACCESS_DENIED_ERROR — access denied for user
		1142: // ER_TABLEACCESS_DENIED_ERROR — command denied to user
		return true
	}
	return false
}

// isIgnorableDDLError reports whether a MySQL error means the object the
// statement creates/drops is already in the intended state — safe to skip when
// re-applying a forward-only migration.
func isIgnorableDDLError(err error) bool {
	var me *mysqldriver.MySQLError
	if !errors.As(err, &me) {
		return false
	}
	switch me.Number {
	case 1050, // ER_TABLE_EXISTS_ERROR — CREATE TABLE on an existing table
		1051, // ER_BAD_TABLE_ERROR — DROP TABLE on a missing table
		1060, // ER_DUP_FIELDNAME — ADD COLUMN already present
		1061, // ER_DUP_KEYNAME — CREATE INDEX already present
		1091: // ER_CANT_DROP_FIELD_OR_KEY — DROP of something that doesn't exist
		return true
	}
	return false
}