chore: name the database in every migration log line
An operator reported migrations being very slow and sent a log with three interleaved runs — and no way to tell one database migrated three times from three databases migrated once. Each line now carries its target: the SQLite file name, mysql:<database>, or baseline:memory for the in-memory pass that derives the schema for a FRESH MySQL database (that one is fast and is not a real database — in the report it sat between two slow runs and looked like a third). A pass also announces how many migrations it will apply, so a run that applies nothing is visibly distinct from one that does.
This commit is contained in:
+21
-6
@@ -6,6 +6,7 @@ import (
|
|||||||
"embed"
|
"embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -161,7 +162,7 @@ func Open(path string) (*sql.DB, error) {
|
|||||||
return nil, fmt.Errorf("ping sqlite: %w", err)
|
return nil, fmt.Errorf("ping sqlite: %w", err)
|
||||||
}
|
}
|
||||||
Dialect = "sqlite"
|
Dialect = "sqlite"
|
||||||
if err := migrate(conn, nil, path); err != nil {
|
if err := migrate(conn, nil, path, filepath.Base(path)); err != nil {
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -175,8 +176,8 @@ var LogSink func(format string, args ...any)
|
|||||||
|
|
||||||
// logMigration records a migration that has just been applied, and how long it
|
// logMigration records a migration that has just been applied, and how long it
|
||||||
// took — the only trace an operator has that a data-rewriting migration ran.
|
// took — the only trace an operator has that a data-rewriting migration ran.
|
||||||
func logMigration(name string, start time.Time) {
|
func logMigration(label, name string, start time.Time) {
|
||||||
logf("db: migration %s applied in %s", name, time.Since(start).Round(time.Millisecond))
|
logf("db[%s]: migration %s applied in %s", label, name, time.Since(start).Round(time.Millisecond))
|
||||||
}
|
}
|
||||||
|
|
||||||
func logf(format string, args ...any) {
|
func logf(format string, args ...any) {
|
||||||
@@ -231,7 +232,11 @@ func backupBeforeRewrite(conn *sql.DB, dbPath, migration string) {
|
|||||||
// skipping those already applied. Intentionally minimal in-house system
|
// skipping those already applied. Intentionally minimal in-house system
|
||||||
// (no external dependency). translate, when non-nil, rewrites each statement
|
// (no external dependency). translate, when non-nil, rewrites each statement
|
||||||
// for a non-SQLite backend (see mysqlDDL); nil means run the SQLite DDL as-is.
|
// for a non-SQLite backend (see mysqlDDL); nil means run the SQLite DDL as-is.
|
||||||
func migrate(conn *sql.DB, translate func(string) string, dbPath string) error {
|
// label names the database being migrated, for the log. Without it three
|
||||||
|
// interleaved migration runs in one log were indistinguishable — an operator
|
||||||
|
// reported "migrations are very slow" and the lines gave no way to tell one
|
||||||
|
// database migrated three times from three databases migrated once.
|
||||||
|
func migrate(conn *sql.DB, translate func(string) string, dbPath, label string) error {
|
||||||
// A non-nil translator means this is the MySQL connection (use the
|
// A non-nil translator means this is the MySQL connection (use the
|
||||||
// per-statement, FK-aware path); nil means a SQLite connection. This is
|
// per-statement, FK-aware path); nil means a SQLite connection. This is
|
||||||
// determined by the caller's argument, NOT the global Dialect, so the
|
// determined by the caller's argument, NOT the global Dialect, so the
|
||||||
@@ -274,6 +279,16 @@ func migrate(conn *sql.DB, translate func(string) string, dbPath string) error {
|
|||||||
return fmt.Errorf("read applied migrations: %w", err)
|
return fmt.Errorf("read applied migrations: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pending := 0
|
||||||
|
for _, name := range names {
|
||||||
|
if !applied[name] {
|
||||||
|
pending++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pending > 0 {
|
||||||
|
logf("db[%s]: %d migration(s) to apply", label, pending)
|
||||||
|
}
|
||||||
|
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
if applied[name] {
|
if applied[name] {
|
||||||
continue // already applied
|
continue // already applied
|
||||||
@@ -307,7 +322,7 @@ func migrate(conn *sql.DB, translate func(string) string, dbPath string) error {
|
|||||||
if _, err := conn.Exec(`INSERT INTO schema_migrations(name) VALUES(?)`, name); err != nil {
|
if _, err := conn.Exec(`INSERT INTO schema_migrations(name) VALUES(?)`, name); err != nil {
|
||||||
return fmt.Errorf("record migration %s: %w", name, err)
|
return fmt.Errorf("record migration %s: %w", name, err)
|
||||||
}
|
}
|
||||||
logMigration(name, start)
|
logMigration(label, name, start)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -327,7 +342,7 @@ func migrate(conn *sql.DB, translate func(string) string, dbPath string) error {
|
|||||||
if err := tx.Commit(); err != nil {
|
if err := tx.Commit(); err != nil {
|
||||||
return fmt.Errorf("commit migration %s: %w", name, err)
|
return fmt.Errorf("commit migration %s: %w", name, err)
|
||||||
}
|
}
|
||||||
logMigration(name, start)
|
logMigration(label, name, start)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -139,6 +139,7 @@ func translateTextColumns(s string) string {
|
|||||||
// app-lifetime ctx with no per-query timeout) hold a connection for the whole
|
// app-lifetime ctx with no per-query timeout) hold a connection for the whole
|
||||||
// chain of remote-MySQL latency, so a handful of concurrent UI bursts drained
|
// chain of remote-MySQL latency, so a handful of concurrent UI bursts drained
|
||||||
// the pool and "after a while nothing updated" — grid, chat, live_status froze.
|
// the pool and "after a while nothing updated" — grid, chat, live_status froze.
|
||||||
|
//
|
||||||
// 40 fits the actual deployment (max 5 ops on a server with max_connections=300 →
|
// 40 fits the actual deployment (max 5 ops on a server with max_connections=300 →
|
||||||
// 40*5 = 200, leaving ~100 for phpMyAdmin/server overhead). This is the per-INSTANCE
|
// 40*5 = 200, leaving ~100 for phpMyAdmin/server overhead). This is the per-INSTANCE
|
||||||
// pool, so keep max_connections >= pool*ops + headroom.
|
// pool, so keep max_connections >= pool*ops + headroom.
|
||||||
@@ -200,7 +201,7 @@ func OpenMySQL(c MySQLConfig) (*sql.DB, error) {
|
|||||||
err = applyMySQLBaseline(conn)
|
err = applyMySQLBaseline(conn)
|
||||||
} else {
|
} else {
|
||||||
// Existing database: apply only the migrations it's missing.
|
// Existing database: apply only the migrations it's missing.
|
||||||
err = migrate(conn, mysqlDDL, "")
|
err = migrate(conn, mysqlDDL, "", "mysql:"+name)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
@@ -287,7 +288,11 @@ func applyMySQLBaseline(conn *sql.DB) error {
|
|||||||
return fmt.Errorf("open baseline sqlite: %w", err)
|
return fmt.Errorf("open baseline sqlite: %w", err)
|
||||||
}
|
}
|
||||||
defer mem.Close()
|
defer mem.Close()
|
||||||
if err := migrate(mem, nil, ""); err != nil {
|
// In-memory SQLite, used only to derive the final schema for a FRESH MySQL
|
||||||
|
// database. Labelled so its (fast) migration lines are not mistaken for a
|
||||||
|
// real database being migrated — in one operator's log this pass sat between
|
||||||
|
// two slow MySQL runs and looked like a third database.
|
||||||
|
if err := migrate(mem, nil, "", "baseline:memory"); err != nil {
|
||||||
return fmt.Errorf("build baseline schema: %w", err)
|
return fmt.Errorf("build baseline schema: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user