From 67203cd4a8e0c41827006527c3486be4b91289f2 Mon Sep 17 00:00:00 2001 From: Gregory Salaun Date: Sun, 14 Jun 2026 00:55:27 +0200 Subject: [PATCH] up --- .claude/settings.json | 7 +- app.go | 162 +++++++++------ frontend/src/components/DvkPanel.tsx | 2 +- frontend/src/components/SettingsModal.tsx | 125 +++++++---- frontend/wailsjs/go/main/App.d.ts | 2 + frontend/wailsjs/go/main/App.js | 4 + frontend/wailsjs/go/models.ts | 16 ++ internal/awardref/awardref.go | 4 +- internal/db/db.go | 64 +++++- internal/db/mysql.go | 242 ++++++++++++++++++++++ internal/db/mysql_test.go | 153 ++++++++++++++ internal/integrations/udp/config.go | 6 +- internal/lookup/lookup.go | 35 ++-- internal/operating/operating.go | 10 +- internal/qso/qso.go | 137 ++++++++---- internal/settings/settings.go | 140 +++++++++---- 16 files changed, 897 insertions(+), 212 deletions(-) create mode 100644 internal/db/mysql.go create mode 100644 internal/db/mysql_test.go diff --git a/.claude/settings.json b/.claude/settings.json index 363b454..e70181e 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -17,7 +17,12 @@ "Bash(gofmt -w internal/ultrabeam/ultrabeam.go)", "Bash(cd \"c:/Perso/Seafile/Programmation/Golang/OpsLog/frontend\" && npx tsc --noEmit 2>&1 | grep -v \"npm notice\" | head && cd .. && /c/Users/legre/go/bin/wails build 2>&1 | tail -2 && ls -la --time-style=+%H:%M build/bin/OpsLog.exe)", "Read(//c/Perso/Seafile/Programmation/Golang/**)", - "Bash(gofmt -w internal/qslcard/*.go)" + "Bash(gofmt -w internal/qslcard/*.go)", + "Bash(awk '{print $3}')", + "Bash(xargs grep -n \"CREATE TABLE\\\\|key\\\\|value\")", + "Bash(ls -la .claude/)", + "Bash(cat .claude/settings.local.json)", + "Bash(cat .claude/settings.json)" ] } } diff --git a/app.go b/app.go index 8aaa7f8..b5efdc8 100644 --- a/app.go +++ b/app.go @@ -369,6 +369,9 @@ type App struct { pttGen int64 // bumped on every key; a delayed unkey only fires if unchanged (guards against a stale release cutting a new transmission) startupErr string // captured for surfacing to the frontend dbPath string // active database file (may be a user-chosen location) + logDb *sql.DB // QSO logbook connection — MySQL when the shared backend is enabled, else == db (local SQLite) + dbBackend string // "sqlite" | "mysql" — the logbook backend actually opened at startup + dbBackendErr string // non-empty when a configured MySQL backend failed and we fell back to SQLite dataDir string // /data — holds config.json, logs, cty.dat migratedFromAppData bool // true when we auto-copied AppData on first portable launch @@ -540,6 +543,11 @@ func (a *App) startup(ctx context.Context) { // to a separate cat.log in the old HamLog folder, which users couldn't find). cat.LogSink = applog.Printf applog.Printf("startup: data dir = %s", dataDir) + // The local SQLite file ALWAYS holds per-operator configuration — settings, + // station profiles, rigs/antennas, cluster nodes, UDP, QSL templates, award + // lists, the lookup cache. Only the QSO logbook itself may live on a shared + // MySQL server (the multi-operator feature). Keeping config local means it + // stays instant even when the shared logbook is on a far-away MySQL. conn, err := db.Open(a.dbPath) if err != nil { a.startupErr = "cannot open db: " + err.Error() @@ -547,7 +555,32 @@ func (a *App) startup(ctx context.Context) { return } a.db = conn - a.qso = qso.NewRepo(conn) + + // Choose where the QSO logbook lives. On a MySQL failure we fall back to the + // local SQLite logbook so the operator can still log (and fix the config). + logbookConn := conn + if mb := readBootstrap(dataDir).MySQL; mb != nil && mb.Enabled { + applog.Printf("startup: logbook backend = MySQL (%s:%d/%s)", mb.Host, mb.Port, mb.Database) + mysqlConn, mErr := db.OpenMySQL(db.MySQLConfig{ + Host: mb.Host, Port: mb.Port, User: mb.User, Password: mb.Password, Database: mb.Database, + }) + if mErr != nil { + applog.Printf("startup: MySQL open failed (%v) — falling back to SQLite logbook", mErr) + a.dbBackendErr = "MySQL: " + mErr.Error() + } else { + logbookConn = mysqlConn + a.dbBackend = "mysql" + } + } + if a.dbBackend == "" { + a.dbBackend = "sqlite" + } + // db.Dialect describes the LOGBOOK backend — the only place SQL actually + // varies (qso JSON extraction). Config repos always run on SQLite. + db.SetDialect(a.dbBackend) + applog.Printf("startup: logbook backend = %s", a.dbBackend) + a.logDb = logbookConn + a.qso = qso.NewRepo(logbookConn) a.settings = settings.NewStore(conn) a.settings.SetSensitivePredicate(isSensitiveSetting) // encrypt passwords at rest when a passphrase is set a.profiles = profile.NewRepo(conn) @@ -882,6 +915,9 @@ func (a *App) shutdown(ctx context.Context) { if a.qsoRec != nil { a.qsoRec.Stop() } + if a.logDb != nil && a.logDb != a.db { + _ = a.logDb.Close() // shared MySQL logbook (separate from the local config DB) + } if a.db != nil { _ = a.db.Close() } @@ -981,32 +1017,49 @@ func copyFileData(src, dst string) error { // ── Database location (config.json pointer) ──────────────────────────── // dbPointer is the tiny bootstrap config stored in the data dir. It must -// live outside the database because we read it to decide which DB to open. +// live outside the database because we read it BEFORE opening any DB to decide +// which backend to use: a local SQLite file (DBPath) or a shared MySQL server +// (MySQL). The MySQL connection lives here — not in the settings table — for +// the same reason: we need it to choose and open the backend at startup. type dbPointer struct { - DBPath string `json:"db_path"` + DBPath string `json:"db_path"` + MySQL *MySQLSettings `json:"mysql,omitempty"` } func dbPointerPath(dataDir string) string { return filepath.Join(dataDir, "config.json") } -// readDBPointer returns the user-chosen DB path, or "" for the default. -func readDBPointer(dataDir string) string { +// readBootstrap returns the full bootstrap config (DB path + MySQL), or a zero +// value if the file is missing/unreadable. +func readBootstrap(dataDir string) dbPointer { + var c dbPointer b, err := os.ReadFile(dbPointerPath(dataDir)) if err != nil { - return "" + return c } - var c dbPointer - if json.Unmarshal(b, &c) != nil { - return "" - } - return strings.TrimSpace(c.DBPath) + _ = json.Unmarshal(b, &c) + c.DBPath = strings.TrimSpace(c.DBPath) + return c } -// writeDBPointer persists the chosen DB path ("" resets to default). -func writeDBPointer(dataDir, path string) error { - b, _ := json.MarshalIndent(dbPointer{DBPath: strings.TrimSpace(path)}, "", " ") +func writeBootstrap(dataDir string, c dbPointer) error { + c.DBPath = strings.TrimSpace(c.DBPath) + b, _ := json.MarshalIndent(c, "", " ") return os.WriteFile(dbPointerPath(dataDir), b, 0o644) } +// readDBPointer returns the user-chosen DB path, or "" for the default. +func readDBPointer(dataDir string) string { + return readBootstrap(dataDir).DBPath +} + +// writeDBPointer persists the chosen DB path ("" resets to default), keeping +// any saved MySQL config intact. +func writeDBPointer(dataDir, path string) error { + c := readBootstrap(dataDir) + c.DBPath = strings.TrimSpace(path) + return writeBootstrap(dataDir, c) +} + // DatabaseSettings describes the active database file for the Settings UI. type DatabaseSettings struct { Path string `json:"path"` @@ -1032,62 +1085,51 @@ type MySQLSettings struct { Database string `json:"database"` } -const ( - keyMySQLEnabled = "mysql.enabled" - keyMySQLHost = "mysql.host" - keyMySQLPort = "mysql.port" - keyMySQLUser = "mysql.user" - keyMySQLPassword = "mysql.password" - keyMySQLDatabase = "mysql.database" -) +// DBBackendStatus reports which backend OpsLog actually opened at startup so +// the Settings UI can confirm the shared MySQL connection (or explain a +// fallback to SQLite when the configured server was unreachable). +type DBBackendStatus struct { + Active string `json:"active"` // "sqlite" | "mysql" + Fallback bool `json:"fallback"` // MySQL was enabled but failed, so we used SQLite + Error string `json:"error"` // the MySQL open error, when Fallback is true +} -// GetMySQLSettings returns the stored shared-database config (defaults applied). +// GetDBBackendStatus returns the active backend and any MySQL fallback error. +func (a *App) GetDBBackendStatus() DBBackendStatus { + return DBBackendStatus{ + Active: a.dbBackend, + Fallback: a.dbBackendErr != "", + Error: a.dbBackendErr, + } +} + +// GetMySQLSettings returns the stored shared-database config from the bootstrap +// file (config.json), with defaults applied. Read before the DB is open, so it +// must not depend on the settings table. func (a *App) GetMySQLSettings() (MySQLSettings, error) { out := MySQLSettings{Port: 3306} - if a.settings == nil { - return out, nil + if mb := readBootstrap(a.dataDir).MySQL; mb != nil { + out = *mb + if out.Port <= 0 { + out.Port = 3306 + } } - m, err := a.settings.GetMany(a.ctx, keyMySQLEnabled, keyMySQLHost, keyMySQLPort, keyMySQLUser, keyMySQLPassword, keyMySQLDatabase) - if err != nil { - return out, err - } - out.Enabled = m[keyMySQLEnabled] == "1" - out.Host = m[keyMySQLHost] - if p, _ := strconv.Atoi(m[keyMySQLPort]); p > 0 { - out.Port = p - } - out.User = m[keyMySQLUser] - out.Password = m[keyMySQLPassword] - out.Database = m[keyMySQLDatabase] return out, nil } -// SaveMySQLSettings persists the shared-database config. (Switching the active -// backend takes effect on restart — wired in a later phase.) +// SaveMySQLSettings persists the shared-database config to the bootstrap file. +// Switching the active backend takes effect on the next launch (we read this +// file before opening any database). func (a *App) SaveMySQLSettings(s MySQLSettings) error { - if a.settings == nil { - return fmt.Errorf("db not initialized") - } if s.Port <= 0 { s.Port = 3306 } - enabled := "0" - if s.Enabled { - enabled = "1" - } - for k, v := range map[string]string{ - keyMySQLEnabled: enabled, - keyMySQLHost: strings.TrimSpace(s.Host), - keyMySQLPort: strconv.Itoa(s.Port), - keyMySQLUser: strings.TrimSpace(s.User), - keyMySQLPassword: s.Password, - keyMySQLDatabase: strings.TrimSpace(s.Database), - } { - if err := a.settings.Set(a.ctx, k, v); err != nil { - return err - } - } - return nil + s.Host = strings.TrimSpace(s.Host) + s.User = strings.TrimSpace(s.User) + s.Database = strings.TrimSpace(s.Database) + c := readBootstrap(a.dataDir) + c.MySQL = &s + return writeBootstrap(a.dataDir, c) } // TestMySQLConnection pings the shared MySQL database with the given settings @@ -4878,7 +4920,7 @@ func (a *App) runDownloadConfirmations(svc extsvc.Service, cfg extsvc.ExternalSe // Ids already QRZ-confirmed locally → "ALREADY CONFIRMED" vs "UPDATED", // without a per-record DB read. alreadyQrz := map[int64]bool{} - if rs, e := a.db.QueryContext(ctx, `SELECT id FROM qso WHERE qrzcom_qso_download_status = 'Y'`); e == nil { + if rs, e := a.logDb.QueryContext(ctx, `SELECT id FROM qso WHERE qrzcom_qso_download_status = 'Y'`); e == nil { for rs.Next() { var id int64 if rs.Scan(&id) == nil { diff --git a/frontend/src/components/DvkPanel.tsx b/frontend/src/components/DvkPanel.tsx index 6257ec5..0aac340 100644 --- a/frontend/src/components/DvkPanel.tsx +++ b/frontend/src/components/DvkPanel.tsx @@ -24,7 +24,7 @@ export function DvkPanel({ messages, status, onPlay, onStop, onClose }: Props) { Voice keyer - {status.playing && transmitting…} + {status.playing && tx...}
- - {mysqlMsg} -
- +
+ + + {mysqlMsg} +
+ {mysqlMsg.startsWith('Saved') && ( +
+ Saved. OpsLog will use the shared MySQL database after a restart. + +
)} + )} {/* Data location */}
diff --git a/frontend/wailsjs/go/main/App.d.ts b/frontend/wailsjs/go/main/App.d.ts index a3a05f0..bf712ae 100644 --- a/frontend/wailsjs/go/main/App.d.ts +++ b/frontend/wailsjs/go/main/App.d.ts @@ -135,6 +135,8 @@ export function GetClusterStatus():Promise>; export function GetCtyDatInfo():Promise; +export function GetDBBackendStatus():Promise; + export function GetDVKMessages():Promise>; export function GetDVKStatus():Promise; diff --git a/frontend/wailsjs/go/main/App.js b/frontend/wailsjs/go/main/App.js index ca7e930..126fc2c 100644 --- a/frontend/wailsjs/go/main/App.js +++ b/frontend/wailsjs/go/main/App.js @@ -242,6 +242,10 @@ export function GetCtyDatInfo() { return window['go']['main']['App']['GetCtyDatInfo'](); } +export function GetDBBackendStatus() { + return window['go']['main']['App']['GetDBBackendStatus'](); +} + export function GetDVKMessages() { return window['go']['main']['App']['GetDVKMessages'](); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 4ee2ac1..f21c585 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -846,6 +846,22 @@ export namespace main { this.file_mod_time = source["file_mod_time"]; } } + export class DBBackendStatus { + active: string; + fallback: boolean; + error: string; + + static createFrom(source: any = {}) { + return new DBBackendStatus(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.active = source["active"]; + this.fallback = source["fallback"]; + this.error = source["error"]; + } + } export class DVKMessage { slot: number; label: string; diff --git a/internal/awardref/awardref.go b/internal/awardref/awardref.go index 6dd48fa..89c7a81 100644 --- a/internal/awardref/awardref.go +++ b/internal/awardref/awardref.go @@ -94,7 +94,7 @@ func (r *Repo) ReplaceAll(ctx context.Context, awardCode string, refs []Ref) (in return 0, fmt.Errorf("clear refs: %w", err) } stmt, err := tx.PrepareContext(ctx, - `INSERT OR REPLACE INTO award_references + `REPLACE INTO award_references (award_code, ref_code, name, dxcc, grp, subgrp, dxcc_list, pattern, valid, valid_from, valid_to, score, bonus, gridsquare, alias) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`) if err != nil { @@ -257,7 +257,7 @@ func (r *Repo) Upsert(ctx context.Context, awardCode string, ref Ref) error { return fmt.Errorf("empty award or reference code") } _, err := r.db.ExecContext(ctx, - `INSERT OR REPLACE INTO award_references + `REPLACE INTO award_references (award_code, ref_code, name, dxcc, grp, subgrp, dxcc_list, pattern, valid, valid_from, valid_to, score, bonus, gridsquare, alias) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, code, rc, strings.TrimSpace(ref.Name), ref.DXCC, ref.Group, ref.SubGrp, diff --git a/internal/db/db.go b/internal/db/db.go index 269448c..2eb5d03 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -81,6 +81,34 @@ func PingMySQL(c MySQLConfig) error { //go:embed migrations/*.sql var migrationsFS embed.FS +// Dialect names the backend opened at startup. Repos consult it for the few +// places where SQLite and MySQL SQL genuinely differ (upserts, JSON extraction). +// Timestamps are generated in Go (NowISO) for both, so most queries are shared. +var Dialect = "sqlite" + +// IsMySQL reports whether the active LOGBOOK backend is the shared MySQL +// server. Config tables always live in local SQLite, so this only governs +// qso-table SQL (see columnExpr). +func IsMySQL() bool { return Dialect == "mysql" } + +// SetDialect pins the logbook dialect ("mysql" | "sqlite"). Called once at +// startup after the logbook backend is chosen, so it doesn't depend on the +// order in which the local and logbook connections were opened. +func SetDialect(d string) { + if d == "mysql" { + Dialect = "mysql" + } else { + Dialect = "sqlite" + } +} + +// NowISO returns the current UTC time as the ISO-8601 string OpsLog stores in +// *_at / date columns — millisecond precision, matching the old SQLite +// strftime('%Y-%m-%dT%H:%M:%fZ','now') default. Bind this as a parameter so the +// same INSERT/UPDATE works on both backends. +func NowISO() string { return time.Now().UTC().Format("2006-01-02T15:04:05.000Z") } + + // Open opens (and creates if needed) the SQLite database at the given path, // enables performance PRAGMAs, and applies embedded migrations. func Open(path string) (*sql.DB, error) { @@ -97,7 +125,8 @@ func Open(path string) (*sql.DB, error) { _ = conn.Close() return nil, fmt.Errorf("ping sqlite: %w", err) } - if err := migrate(conn); err != nil { + Dialect = "sqlite" + if err := migrate(conn, nil); err != nil { _ = conn.Close() return nil, err } @@ -106,12 +135,16 @@ func Open(path string) (*sql.DB, error) { // migrate applies all embedded *.sql migrations in alphabetical order, // skipping those already applied. Intentionally minimal in-house system -// (no external dependency). -func migrate(conn *sql.DB) error { - if _, err := conn.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( +// (no external dependency). translate, when non-nil, rewrites each statement +// for a non-SQLite backend (see mysqlDDL); nil means run the SQLite DDL as-is. +func migrate(conn *sql.DB, translate func(string) string) error { + if translate == nil { + translate = func(s string) string { return s } + } + if _, err := conn.Exec(translate(`CREATE TABLE IF NOT EXISTS schema_migrations ( name TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) - )`); err != nil { + )`)); err != nil { return fmt.Errorf("create schema_migrations: %w", err) } @@ -141,11 +174,30 @@ func migrate(conn *sql.DB) error { if err != nil { return fmt.Errorf("read migration %s: %w", name, err) } + sqlText := translate(string(content)) + + // MySQL implicitly commits each DDL statement, so a wrapping transaction + // gives no atomicity — a mid-file failure would leave columns/tables + // behind, unrecorded, and every restart would re-run and choke on + // "duplicate column". Instead apply statement-by-statement and tolerate + // "already exists" errors, making migrations idempotent (and self-healing + // for a previously half-applied database). + if IsMySQL() { + if err := applyMySQLMigration(conn, sqlText); err != nil { + return fmt.Errorf("apply migration %s: %w", name, err) + } + if _, err := conn.Exec(`INSERT INTO schema_migrations(name) VALUES(?)`, name); err != nil { + return fmt.Errorf("record migration %s: %w", name, err) + } + continue + } + + // SQLite supports DDL inside a transaction, so keep it atomic there. tx, err := conn.Begin() if err != nil { return fmt.Errorf("begin tx for %s: %w", name, err) } - if _, err := tx.Exec(string(content)); err != nil { + if _, err := tx.Exec(sqlText); err != nil { _ = tx.Rollback() return fmt.Errorf("apply migration %s: %w", name, err) } diff --git a/internal/db/mysql.go b/internal/db/mysql.go new file mode 100644 index 0000000..46d8f5f --- /dev/null +++ b/internal/db/mysql.go @@ -0,0 +1,242 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + "time" + + mysqldriver "github.com/go-sql-driver/mysql" +) + +// ── SQLite → MySQL schema translation ────────────────────────────────── +// +// OpsLog's migrations are authored in SQLite dialect (the default backend). +// For the shared-MySQL backend we translate that DDL on the fly rather than +// maintaining a second set of migration files, so the two backends can never +// drift apart. The translation is deliberately narrow — it only rewrites the +// SQLite-isms that actually appear in our migrations: +// +// - INTEGER PRIMARY KEY [AUTOINCREMENT] → BIGINT AUTO_INCREMENT PRIMARY KEY +// - bare INTEGER → BIGINT (so FK child/parent types match) +// - DEFAULT (strftime(... 'now')) → DEFAULT '' (timestamps come from Go) +// - TEXT → VARCHAR(255), except a few free-text +// columns that stay TEXT/LONGTEXT +// - reserved column `key` → backticked +// - CREATE INDEX IF NOT EXISTS → CREATE INDEX (MySQL has no IF NOT EXISTS) +// +// REAL is left as-is (MySQL accepts it as an alias for DOUBLE). CREATE/DROP +// TABLE IF [NOT] EXISTS, RENAME TO, INSERT…SELECT, CHECK(...) and foreign keys +// are all valid MySQL as written. + +var ( + reStrftimeDefault = regexp.MustCompile(`\(strftime\('%Y-%m-%dT%H:%M:%fZ',\s*'now'\)\)`) + reBareInteger = regexp.MustCompile(`\bINTEGER\b`) + reColText = regexp.MustCompile(`(\w+)\s+TEXT\b`) + reKeyColumn = regexp.MustCompile(`(?m)^(\s*)key\b`) +) + +// MySQL's InnoDB row-size limit is 65535 bytes (excluding off-page TEXT/BLOB). +// The qso table has ~150 columns, so making them all VARCHAR(255) (1020 bytes +// each in utf8mb4) overflows the row. We therefore emit TEXT (stored off-page, +// ~20 bytes in-row) for the bulk of columns, and VARCHAR only where a column +// actually needs in-row storage: it's indexed, a primary key, or carries a +// DEFAULT (TEXT can't hold a literal default in MySQL). + +// varcharColumns must be VARCHAR because they're indexed or part of a primary +// key declared on a separate line (so the DEFAULT/PRIMARY-KEY line heuristic +// below can't catch them). MySQL can't index a TEXT column without a prefix +// length. Keyed by column name (a name may repeat across tables — harmless). +var varcharColumns = map[string]bool{ + // qso indexes (0001, 0003, 0019) + "callsign": true, "qso_date": true, "band": true, "mode": true, + "grid": true, "station_callsign": true, "state": true, "contest_id": true, + "sat_name": true, "prop_mode": true, "sig": true, "wwff_ref": true, "skcc": true, + // integrations_udp index (0011) + "direction": true, + // award_references composite primary key (0017) + "award_code": true, "ref_code": true, +} + +// longTextColumns override the default TEXT with a bigger type where the +// content can be large (a full QSL-template document). +var longTextColumns = map[string]string{ + "json": "LONGTEXT", +} + +// mysqlDDL rewrites a SQLite DDL/DML statement (or multi-statement migration +// file) into the MySQL dialect. See the package note above for the rules. +func mysqlDDL(stmt string) string { + s := stmt + // MySQL has no IF NOT EXISTS for CREATE INDEX (migrations run once anyway). + s = strings.ReplaceAll(s, "CREATE INDEX IF NOT EXISTS", "CREATE INDEX") + // Auto-increment primary keys. Both the AUTOINCREMENT form and the bare + // "INTEGER PRIMARY KEY" (SQLite rowid alias) must become AUTO_INCREMENT so + // inserts that omit the id still get one. + s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY AUTOINCREMENT", "BIGINT AUTO_INCREMENT PRIMARY KEY") + s = strings.ReplaceAll(s, "INTEGER PRIMARY KEY", "BIGINT AUTO_INCREMENT PRIMARY KEY") + // Timestamp defaults: Go supplies ISO strings, so drop the SQLite function + // default (TEXT/VARCHAR can't carry it in MySQL anyway). + s = reStrftimeDefault.ReplaceAllString(s, "''") + // Remaining INTEGER columns → BIGINT so foreign-key parent/child types match. + s = reBareInteger.ReplaceAllString(s, "BIGINT") + // TEXT columns → TEXT (off-page) by default, VARCHAR(255) only when indexed, + // a primary key, or default-bearing — keeping the row under MySQL's limit. + s = translateTextColumns(s) + // `key` is a MySQL reserved word — backtick the settings column declaration. + s = reKeyColumn.ReplaceAllString(s, "$1`key`") + return s +} + +// translateTextColumns rewrites each " TEXT" column declaration to the +// right MySQL type, deciding per line so it can see whether the line carries a +// DEFAULT or an inline PRIMARY KEY. See varcharColumns / longTextColumns. +func translateTextColumns(s string) string { + lines := strings.Split(s, "\n") + for i, line := range lines { + m := reColText.FindStringSubmatchIndex(line) + if m == nil { + continue + } + col := line[m[2]:m[3]] + var typ string + switch { + case longTextColumns[col] != "": + typ = longTextColumns[col] + case varcharColumns[col], + strings.Contains(line, "DEFAULT"), + strings.Contains(line, "PRIMARY KEY"): + typ = "VARCHAR(255)" + default: + typ = "TEXT" + } + lines[i] = line[:m[0]] + col + " " + typ + line[m[1]:] + } + return strings.Join(lines, "\n") +} + +// OpenMySQL opens the shared MySQL logbook, creating the database if needed, +// then applies the (translated) embedded migrations. multiStatements is enabled +// so multi-statement migration files run in a single Exec. +func OpenMySQL(c MySQLConfig) (*sql.DB, error) { + if strings.TrimSpace(c.Host) == "" { + return nil, fmt.Errorf("host is required") + } + name := strings.TrimSpace(c.Database) + if !validDBIdent(name) { + return nil, fmt.Errorf("invalid database name %q (letters, digits, underscore only)", name) + } + // Ensure the database exists (connect server-level first). + if err := PingMySQL(c); err != nil { + return nil, err + } + conn, err := sql.Open("mysql", c.dsn()) + if err != nil { + return nil, fmt.Errorf("open mysql: %w", err) + } + // The UI fires bursts of concurrent queries (the Preferences dialog alone + // loads ~8 settings in parallel, plus the grid and live cluster status). A + // low cap turns such a burst into a deadlock-like stall against a remote + // server, so keep a generous pool with idle reaping. SQLite ran uncapped. + conn.SetMaxOpenConns(50) + conn.SetMaxIdleConns(10) + conn.SetConnMaxIdleTime(90 * time.Second) + conn.SetConnMaxLifetime(5 * time.Minute) + if err := conn.Ping(); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("connect to %s: %w", name, err) + } + // Set the dialect before migrating so the runner takes the MySQL path + // (per-statement, idempotent) rather than the SQLite transaction path. + Dialect = "mysql" + if err := migrate(conn, mysqlDDL); err != nil { + _ = conn.Close() + Dialect = "sqlite" // migration failed; we'll fall back to SQLite + return nil, err + } + return conn, nil +} + +// applyMySQLMigration executes a translated migration one statement at a time. +// MySQL has no multi-statement Exec without a special flag and auto-commits DDL, +// so running statements individually (and tolerating "already exists" errors) +// keeps migrations idempotent and lets a half-applied database self-heal. +// +// It runs on a single dedicated connection with FOREIGN_KEY_CHECKS=0, because a +// few migrations recreate tables by DROP/RENAME in an order that trips MySQL's +// foreign-key enforcement (SQLite is laxer). The flag is reset to 1 before the +// connection returns to the pool, so runtime queries keep FK enforcement. +func applyMySQLMigration(conn *sql.DB, sqlText string) error { + ctx := context.Background() + c, err := conn.Conn(ctx) + if err != nil { + return fmt.Errorf("acquire conn: %w", err) + } + defer func() { + _, _ = c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=1") + _ = c.Close() + }() + if _, err := c.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS=0"); err != nil { + return fmt.Errorf("disable fk checks: %w", err) + } + for _, stmt := range splitStatements(sqlText) { + if _, err := c.ExecContext(ctx, stmt); err != nil { + if isIgnorableDDLError(err) { + continue + } + return fmt.Errorf("statement %q: %w", firstLine(stmt), err) + } + } + return nil +} + +// splitStatements breaks a migration file into individual SQL statements, +// dropping full-line comments and blank fragments. Our migrations never embed +// a ';' inside a string literal, so a simple split is safe. +func splitStatements(sqlText string) []string { + var b strings.Builder + for _, line := range strings.Split(sqlText, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + b.WriteString(line) + b.WriteByte('\n') + } + var out []string + for _, part := range strings.Split(b.String(), ";") { + if strings.TrimSpace(part) != "" { + out = append(out, part) + } + } + return out +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +// isIgnorableDDLError reports whether a MySQL error means the object the +// statement creates/drops is already in the intended state — safe to skip when +// re-applying a forward-only migration. +func isIgnorableDDLError(err error) bool { + var me *mysqldriver.MySQLError + if !errors.As(err, &me) { + return false + } + switch me.Number { + case 1050, // ER_TABLE_EXISTS_ERROR — CREATE TABLE on an existing table + 1051, // ER_BAD_TABLE_ERROR — DROP TABLE on a missing table + 1060, // ER_DUP_FIELDNAME — ADD COLUMN already present + 1061, // ER_DUP_KEYNAME — CREATE INDEX already present + 1091: // ER_CANT_DROP_FIELD_OR_KEY — DROP of something that doesn't exist + return true + } + return false +} diff --git a/internal/db/mysql_test.go b/internal/db/mysql_test.go new file mode 100644 index 0000000..5dbf970 --- /dev/null +++ b/internal/db/mysql_test.go @@ -0,0 +1,153 @@ +package db + +import ( + "regexp" + "strings" + "testing" +) + +func TestMySQLDDL_Translations(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"autoinc pk", "id INTEGER PRIMARY KEY AUTOINCREMENT,", "id BIGINT AUTO_INCREMENT PRIMARY KEY,"}, + {"plain pk", "id INTEGER PRIMARY KEY,", "id BIGINT AUTO_INCREMENT PRIMARY KEY,"}, + {"bare integer", "freq_hz INTEGER,", "freq_hz BIGINT,"}, + {"strftime default", "created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),", + "created_at VARCHAR(255) NOT NULL DEFAULT '',"}, + {"strftime tight", "updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),", + "updated_at VARCHAR(255) NOT NULL DEFAULT '',"}, + // The column-name/type whitespace collapses to a single space — harmless. + {"text default N", "qsl_sent TEXT DEFAULT 'N',", "qsl_sent VARCHAR(255) DEFAULT 'N',"}, + {"indexed col → varchar", "callsign TEXT NOT NULL,", "callsign VARCHAR(255) NOT NULL,"}, + {"plain non-indexed col → text", "name TEXT,", "name TEXT,"}, + {"plain text stays text", "comment TEXT,", "comment TEXT,"}, + {"json longtext", " json TEXT NOT NULL,", " json LONGTEXT NOT NULL,"}, + {"create index", "CREATE INDEX IF NOT EXISTS idx_qso_dxcc ON qso(dxcc);", + "CREATE INDEX idx_qso_dxcc ON qso(dxcc);"}, + } + for _, c := range cases { + if got := mysqlDDL(c.in); got != c.want { + t.Errorf("%s:\n in %q\n got %q\n want %q", c.name, c.in, got, c.want) + } + } +} + +func TestMySQLDDL_KeyColumnBackticked(t *testing.T) { + in := "CREATE TABLE IF NOT EXISTS settings (\n key TEXT PRIMARY KEY,\n value TEXT NOT NULL\n);" + got := mysqlDDL(in) + if !strings.Contains(got, "`key` VARCHAR(255) PRIMARY KEY") { + t.Errorf("key column not backticked/translated:\n%s", got) + } +} + +func TestSplitStatements(t *testing.T) { + in := "-- a comment\n" + + "ALTER TABLE qso ADD COLUMN a TEXT;\n" + + "ALTER TABLE qso ADD COLUMN b TEXT; -- inline note\n" + + "\n" + + "CREATE INDEX idx ON qso(a);\n" + got := splitStatements(in) + if len(got) != 3 { + t.Fatalf("want 3 statements, got %d: %#v", len(got), got) + } + if !strings.Contains(got[0], "ADD COLUMN a") || + !strings.Contains(got[1], "ADD COLUMN b") || + !strings.Contains(got[2], "CREATE INDEX") { + t.Errorf("unexpected split: %#v", got) + } + // No fragment should be a comment-only or blank statement. + for _, s := range got { + if strings.TrimSpace(s) == "" { + t.Errorf("empty statement in result: %#v", got) + } + } +} + +// Every embedded migration must split into at least one runnable statement +// and never produce an empty fragment. +func TestSplitStatements_AllMigrations(t *testing.T) { + entries, _ := migrationsFS.ReadDir("migrations") + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".sql") { + continue + } + raw, _ := migrationsFS.ReadFile("migrations/" + e.Name()) + stmts := splitStatements(mysqlDDL(string(raw))) + if len(stmts) == 0 { + t.Errorf("%s: produced no statements", e.Name()) + } + for _, s := range stmts { + if strings.TrimSpace(s) == "" { + t.Errorf("%s: empty statement fragment", e.Name()) + } + } + } +} + +// TestMySQLDDL_QSORowSizeUnderLimit guards against the InnoDB 65535-byte row +// limit: every VARCHAR(255) in utf8mb4 costs 1020 bytes in-row, and the qso +// table (built from 0001 + the qso-only ALTERs in 0003 and 0019) must stay well +// clear of it. TEXT columns are off-page and don't count here. +func TestMySQLDDL_QSORowSizeUnderLimit(t *testing.T) { + qsoMigrations := []string{"0001_init.sql", "0003_adif_extra.sql", "0019_adif_317_fields.sql"} + varchars := 0 + for _, name := range qsoMigrations { + raw, err := migrationsFS.ReadFile("migrations/" + name) + if err != nil { + t.Fatal(err) + } + varchars += strings.Count(mysqlDDL(string(raw)), "VARCHAR(255)") + } + const bytesPerVarchar = 255 * 4 // utf8mb4 + rowBytes := varchars * bytesPerVarchar + t.Logf("qso VARCHAR(255) columns: %d (~%d bytes in-row)", varchars, rowBytes) + if rowBytes > 60000 { // leave headroom under the 65535 hard limit + t.Errorf("qso row too large: %d VARCHAR(255) cols = ~%d bytes (limit 65535)", varchars, rowBytes) + } +} + +// TestMySQLDDL_NoLeftoverSQLiteisms translates every embedded migration and +// fails if any SQLite-only construct survives — a fast guard against a new +// migration sneaking in a dialect-ism the translator doesn't cover. +func TestMySQLDDL_NoLeftoverSQLiteisms(t *testing.T) { + entries, err := migrationsFS.ReadDir("migrations") + if err != nil { + t.Fatal(err) + } + reInteger := regexp.MustCompile(`\bINTEGER\b`) + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".sql") { + continue + } + raw, err := migrationsFS.ReadFile("migrations/" + e.Name()) + if err != nil { + t.Fatal(err) + } + out := mysqlDDL(string(raw)) + if strings.Contains(out, "AUTOINCREMENT") { + t.Errorf("%s: AUTOINCREMENT survived", e.Name()) + } + if strings.Contains(out, "strftime") { + t.Errorf("%s: strftime survived", e.Name()) + } + if strings.Contains(out, "IF NOT EXISTS idx") { + t.Errorf("%s: CREATE INDEX IF NOT EXISTS survived", e.Name()) + } + // Strip comment lines before checking for bare INTEGER (comments are + // prose and may legitimately mention the word). + var code strings.Builder + for _, ln := range strings.Split(out, "\n") { + if strings.HasPrefix(strings.TrimSpace(ln), "--") { + continue + } + code.WriteString(ln) + code.WriteByte('\n') + } + if reInteger.MatchString(code.String()) { + t.Errorf("%s: bare INTEGER survived in code", e.Name()) + } + } +} diff --git a/internal/integrations/udp/config.go b/internal/integrations/udp/config.go index 5787882..66ed70a 100644 --- a/internal/integrations/udp/config.go +++ b/internal/integrations/udp/config.go @@ -13,6 +13,8 @@ import ( "context" "database/sql" "fmt" + + "hamlog/internal/db" ) // Direction is "inbound" (we listen) or "outbound" (we emit). @@ -112,10 +114,10 @@ func (r *Repo) Save(ctx context.Context, c *Config) error { direction = ?, name = ?, port = ?, service_type = ?, multicast = ?, multicast_group = ?, destination_ip = ?, enabled = ?, sort_order = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + updated_at = ? WHERE id = ?`, c.Direction, c.Name, c.Port, c.ServiceType, - mc, c.MulticastGroup, c.DestinationIP, en, c.SortOrder, c.ID) + mc, c.MulticastGroup, c.DestinationIP, en, c.SortOrder, db.NowISO(), c.ID) if err != nil { return fmt.Errorf("update udp: %w", err) } diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index d404f3d..2bb9c30 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -12,6 +12,8 @@ import ( "sync" "time" "unicode" + + "hamlog/internal/db" ) // ErrNotFound is returned by providers when a callsign is unknown. @@ -357,24 +359,29 @@ func (c *Cache) Get(ctx context.Context, callsign string) (Result, bool) { return r, true } -// Put upserts a lookup result. +// Put upserts a lookup result. fetched_at is generated in Go (NowISO) so the +// INSERT is backend-agnostic; the conflict tail is dialect-specific. func (c *Cache) Put(ctx context.Context, r Result) error { - _, err := c.db.ExecContext(ctx, ` + updateCols := []string{ + "name", "qth", "address", "state", "cnty", + "country", "grid", "lat", "lon", + "dxcc", "cqz", "ituz", "cont", "email", "qsl_via", "image_url", + "source", "fetched_at", + } + // The lookup cache always lives in the local SQLite database, so SQLite + // upsert syntax is used unconditionally. + sets := make([]string, len(updateCols)) + for i, c := range updateCols { + sets[i] = c + " = excluded." + c + } + q := ` INSERT INTO callsign_cache(callsign, name, qth, address, state, cnty, country, grid, lat, lon, dxcc, cqz, ituz, cont, email, qsl_via, image_url, source, fetched_at) - VALUES(?,?,?,?,?,?, ?,?,?,?, ?,?,?,?,?,?,?, ?, - strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) - ON CONFLICT(callsign) DO UPDATE SET - name = excluded.name, qth = excluded.qth, address = excluded.address, - state = excluded.state, cnty = excluded.cnty, - country = excluded.country, grid = excluded.grid, - lat = excluded.lat, lon = excluded.lon, - dxcc = excluded.dxcc, cqz = excluded.cqz, ituz = excluded.ituz, - cont = excluded.cont, email = excluded.email, qsl_via = excluded.qsl_via, - image_url = excluded.image_url, - source = excluded.source, fetched_at = excluded.fetched_at`, + VALUES(?,?,?,?,?,?, ?,?,?,?, ?,?,?,?,?,?,?, ?,?) + ON CONFLICT(callsign) DO UPDATE SET ` + strings.Join(sets, ", ") + _, err := c.db.ExecContext(ctx, q, r.Callsign, nullable(r.Name), nullable(r.QTH), nullable(r.Address), nullable(r.State), nullable(r.County), nullable(r.Country), nullable(r.Grid), @@ -382,7 +389,7 @@ func (c *Cache) Put(ctx context.Context, r Result) error { nullableInt(r.DXCC), nullableInt(r.CQZ), nullableInt(r.ITUZ), nullable(r.Continent), nullable(r.Email), nullable(r.QSLVia), nullable(r.ImageURL), - r.Source, + r.Source, db.NowISO(), ) return err } diff --git a/internal/operating/operating.go b/internal/operating/operating.go index 41a3318..365fed8 100644 --- a/internal/operating/operating.go +++ b/internal/operating/operating.go @@ -8,6 +8,8 @@ import ( "database/sql" "fmt" "strings" + + "hamlog/internal/db" ) // Station is a radio / TRX line. The display Name is also what gets @@ -190,8 +192,8 @@ func (r *Repo) SaveStation(ctx context.Context, s *Station) error { _, err := r.db.ExecContext(ctx, `UPDATE operating_stations SET name = ?, tx_pwr = ?, sort_order = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') - WHERE id = ?`, s.Name, pwr, s.SortOrder, s.ID) + updated_at = ? + WHERE id = ?`, s.Name, pwr, s.SortOrder, db.NowISO(), s.ID) if err != nil { return fmt.Errorf("update station: %w", err) } @@ -230,8 +232,8 @@ func (r *Repo) SaveAntenna(ctx context.Context, a *Antenna) error { if _, err := tx.ExecContext(ctx, `UPDATE operating_antennas SET name = ?, sort_order = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') - WHERE id = ?`, a.Name, a.SortOrder, a.ID); err != nil { + updated_at = ? + WHERE id = ?`, a.Name, a.SortOrder, db.NowISO(), a.ID); err != nil { return fmt.Errorf("update antenna: %w", err) } } diff --git a/internal/qso/qso.go b/internal/qso/qso.go index 3858f02..cb0c2de 100644 --- a/internal/qso/qso.go +++ b/internal/qso/qso.go @@ -10,6 +10,8 @@ import ( "sort" "strings" "time" + + "hamlog/internal/db" ) // MergeNonZero copies every non-zero field of src onto dst. Zero-value src @@ -255,6 +257,19 @@ var columnCount = countColumns(columnList) // insertPlaceholders returns "?,?,?,..." matching columnCount. var insertPlaceholders = buildInsertPlaceholders() +// insertCols/insertVals append created_at + updated_at so they're set from Go +// (NowISO) on every backend — MySQL has no strftime default, and binding the +// timestamp keeps a single backend-agnostic INSERT. insertArgs pairs with them. +const insertCols = columnList + `, created_at, updated_at` + +var insertVals = insertPlaceholders + ",?,?" + +// insertArgs returns the column values plus the two timestamps for an INSERT. +func (q *QSO) insertArgs() []any { + now := db.NowISO() + return append(q.args(), now, now) +} + func countColumns(s string) int { n := 1 for i := 0; i < len(s); i++ { @@ -349,34 +364,30 @@ func (r *Repo) Add(ctx context.Context, q QSO) (int64, error) { q.QSODate = time.Now().UTC() } res, err := r.db.ExecContext(ctx, - `INSERT INTO qso (`+columnList+`) VALUES (`+insertPlaceholders+`)`, - q.args()...) + `INSERT INTO qso (`+insertCols+`) VALUES (`+insertVals+`)`, + q.insertArgs()...) if err != nil { return 0, fmt.Errorf("insert qso: %w", err) } return res.LastInsertId() } -// AddBatch inserts many QSOs inside a single transaction using a prepared -// statement. Empty-callsign records are skipped. Returns rows inserted. +// batchInsertRows is how many QSOs go into one multi-row INSERT on MySQL. Each +// row carries ~135 columns, so 200 rows ≈ 27k bound parameters — well under +// MySQL's 65535-placeholder limit and a modest packet — while cutting network +// round-trips ~200× versus one INSERT per row (critical for a remote server). +const batchInsertRows = 200 + +// AddBatch inserts many QSOs inside a single transaction. Empty-callsign records +// are skipped. On MySQL it uses chunked multi-row INSERTs so a 27k-record import +// over a remote link takes seconds, not many minutes; on local SQLite a prepared +// statement per row is already fast. func (r *Repo) AddBatch(ctx context.Context, qsos []QSO) (int64, error) { if len(qsos) == 0 { return 0, nil } - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return 0, fmt.Errorf("begin tx: %w", err) - } - defer tx.Rollback() - - stmt, err := tx.PrepareContext(ctx, - `INSERT INTO qso (`+columnList+`) VALUES (`+insertPlaceholders+`)`) - if err != nil { - return 0, fmt.Errorf("prepare batch insert: %w", err) - } - defer stmt.Close() - - var inserted int64 + // Normalise and drop empty-callsign records up front. + rows := make([]QSO, 0, len(qsos)) for _, q := range qsos { q.Callsign = strings.ToUpper(strings.TrimSpace(q.Callsign)) if q.Callsign == "" { @@ -385,10 +396,57 @@ func (r *Repo) AddBatch(ctx context.Context, qsos []QSO) (int64, error) { if q.QSODate.IsZero() { q.QSODate = time.Now().UTC() } - if _, err := stmt.ExecContext(ctx, q.args()...); err != nil { - return inserted, fmt.Errorf("insert qso %q: %w", q.Callsign, err) + rows = append(rows, q) + } + if len(rows) == 0 { + return 0, nil + } + + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var inserted int64 + if db.IsMySQL() { + rowPlaceholder := "(" + insertVals + ")" + for start := 0; start < len(rows); start += batchInsertRows { + end := start + batchInsertRows + if end > len(rows) { + end = len(rows) + } + chunk := rows[start:end] + var sb strings.Builder + sb.WriteString("INSERT INTO qso (") + sb.WriteString(insertCols) + sb.WriteString(") VALUES ") + args := make([]any, 0, len(chunk)*(columnCount+2)) + for j := range chunk { + if j > 0 { + sb.WriteByte(',') + } + sb.WriteString(rowPlaceholder) + args = append(args, chunk[j].insertArgs()...) + } + if _, err := tx.ExecContext(ctx, sb.String(), args...); err != nil { + return inserted, fmt.Errorf("batch insert: %w", err) + } + inserted += int64(len(chunk)) + } + } else { + stmt, err := tx.PrepareContext(ctx, + `INSERT INTO qso (`+insertCols+`) VALUES (`+insertVals+`)`) + if err != nil { + return 0, fmt.Errorf("prepare batch insert: %w", err) + } + defer stmt.Close() + for i := range rows { + if _, err := stmt.ExecContext(ctx, rows[i].insertArgs()...); err != nil { + return inserted, fmt.Errorf("insert qso %q: %w", rows[i].Callsign, err) + } + inserted++ } - inserted++ } if err := tx.Commit(); err != nil { return 0, fmt.Errorf("commit batch: %w", err) @@ -455,8 +513,8 @@ func (r *Repo) ListForUpload(ctx context.Context, column, value string) ([]Uploa func (r *Repo) MarkQRZUploaded(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET qrzcom_qso_upload_status = 'Y', qrzcom_qso_upload_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark qrz uploaded %d: %w", id, err) } @@ -468,8 +526,8 @@ func (r *Repo) MarkQRZUploaded(ctx context.Context, id int64, date string) error func (r *Repo) MarkClublogUploaded(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET clublog_qso_upload_status = 'Y', clublog_qso_upload_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark clublog uploaded %d: %w", id, err) } @@ -481,8 +539,8 @@ func (r *Repo) MarkClublogUploaded(ctx context.Context, id int64, date string) e func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET lotw_sent = 'Y', lotw_sent_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark lotw uploaded %d: %w", id, err) } @@ -494,8 +552,8 @@ func (r *Repo) MarkLoTWUploaded(ctx context.Context, id int64, date string) erro func (r *Repo) MarkEQSLSent(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET eqsl_sent = 'Y', eqsl_sent_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark eqsl sent %d: %w", id, err) } @@ -515,9 +573,9 @@ func (r *Repo) Update(ctx context.Context, q QSO) error { q.QSODate = time.Now().UTC() } setClause := buildUpdateSetClause() - args := append(q.args(), q.ID) + args := append(q.args(), db.NowISO(), q.ID) res, err := r.db.ExecContext(ctx, - `UPDATE qso SET `+setClause+`, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, + `UPDATE qso SET `+setClause+`, updated_at = ? WHERE id = ?`, args...) if err != nil { return fmt.Errorf("update qso %d: %w", q.ID, err) @@ -714,6 +772,11 @@ func columnExpr(field string) (string, bool) { return f, true } if key, ok := filterableExtras[f]; ok { + if db.IsMySQL() { + // JSON_EXTRACT errors on an invalid/empty document, so guard with + // NULLIF; JSON_UNQUOTE strips the quotes MySQL adds around strings. + return "JSON_UNQUOTE(JSON_EXTRACT(NULLIF(extras_json,''), '$." + key + "'))", true + } return "json_extract(extras_json, '$." + key + "')", true } return "", false @@ -1348,7 +1411,7 @@ func (r *Repo) Count(ctx context.Context) (int64, error) { // far cheaper than N exists-queries during the import loop. func (r *Repo) ExistingDedupeKeys(ctx context.Context) (map[string]struct{}, error) { rows, err := r.db.QueryContext(ctx, ` - SELECT callsign, strftime('%Y-%m-%dT%H:%M', qso_date), band, mode + SELECT callsign, substr(qso_date, 1, 16), band, mode FROM qso`) if err != nil { return nil, err @@ -1376,7 +1439,7 @@ func DedupeKey(callsign, qsoDateMinute, band, mode string) string { // confirmations back to local QSOs. func (r *Repo) DedupeKeyIDs(ctx context.Context) (map[string]int64, error) { rows, err := r.db.QueryContext(ctx, ` - SELECT id, callsign, strftime('%Y-%m-%dT%H:%M', qso_date), band, mode + SELECT id, callsign, substr(qso_date, 1, 16), band, mode FROM qso`) if err != nil { return nil, err @@ -1463,8 +1526,8 @@ func (r *Repo) ConfirmedSlots(ctx context.Context, cols []string) (ConfirmedSets func (r *Repo) MarkQRZConfirmed(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET qrzcom_qso_download_status = 'Y', qrzcom_qso_download_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark qrz confirmed %d: %w", id, err) } @@ -1476,8 +1539,8 @@ func (r *Repo) MarkQRZConfirmed(ctx context.Context, id int64, date string) erro func (r *Repo) MarkLoTWConfirmed(ctx context.Context, id int64, date string) error { _, err := r.db.ExecContext(ctx, `UPDATE qso SET lotw_rcvd = 'Y', lotw_rcvd_date = ?, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?`, - date, id) + updated_at = ? WHERE id = ?`, + date, db.NowISO(), id) if err != nil { return fmt.Errorf("mark lotw confirmed %d: %w", id, err) } diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 43ccbd4..5e072b2 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -13,6 +13,7 @@ import ( "fmt" "sync" + "hamlog/internal/db" "hamlog/internal/secret" ) @@ -27,10 +28,62 @@ type Store struct { mu sync.RWMutex cipher Cipher // non-nil when secrets are unlocked sensitive func(key string) bool // which keys are encrypted at rest + + // cache holds every setting's RAW (as-stored) value, loaded once. Reads are + // served from memory so the Preferences dialog (dozens of keys) doesn't pay + // a network round-trip per key against a remote MySQL. Decryption still + // happens on read, so a later Unlock takes effect without reloading. + cache map[string]string + cached bool } func NewStore(db *sql.DB) *Store { return &Store{db: db} } +// ensureCache lazily loads all settings into memory on first read. A concurrent +// double-load is harmless (the result is identical), so it's done without a +// long-held lock. +func (s *Store) ensureCache(ctx context.Context) error { + s.mu.RLock() + ok := s.cached + s.mu.RUnlock() + if ok { + return nil + } + rows, err := s.db.QueryContext(ctx, "SELECT `key`, value FROM settings") + if err != nil { + return err + } + defer rows.Close() + m := make(map[string]string, 256) + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + return err + } + m[k] = v + } + if err := rows.Err(); err != nil { + return err + } + s.mu.Lock() + if !s.cached { + s.cache = m + s.cached = true + } + s.mu.Unlock() + return nil +} + +// cachePut updates the in-memory copy after a write so reads stay coherent. +func (s *Store) cachePut(key, raw string) { + s.mu.Lock() + if s.cache == nil { + s.cache = map[string]string{} + } + s.cache[key] = raw + s.mu.Unlock() +} + // SetSensitivePredicate registers which keys hold secrets. Set once at startup. func (s *Store) SetSensitivePredicate(fn func(key string) bool) { s.mu.Lock() @@ -101,39 +154,36 @@ func (s *Store) encodeWrite(key, val string) string { // Get returns the value for key, or "" if not set. func (s *Store) Get(ctx context.Context, key string) (string, error) { - var v string - err := s.db.QueryRowContext(ctx, `SELECT value FROM settings WHERE key = ?`, key).Scan(&v) - if err == sql.ErrNoRows { - return "", nil - } + raw, err := s.GetRaw(ctx, key) if err != nil { return "", err } - return s.decodeRead(key, v), nil + return s.decodeRead(key, raw), nil } // GetRaw returns the stored value WITHOUT decryption — used by the passphrase // migration which must read/re-write the raw ciphertext or plaintext. func (s *Store) GetRaw(ctx context.Context, key string) (string, error) { - var v string - err := s.db.QueryRowContext(ctx, `SELECT value FROM settings WHERE key = ?`, key).Scan(&v) - if err == sql.ErrNoRows { - return "", nil + if err := s.ensureCache(ctx); err != nil { + return "", err } - return v, err + s.mu.RLock() + v := s.cache[key] // "" when absent + s.mu.RUnlock() + return v, nil } // SetRaw stores a value verbatim (no encryption) — used by the migration. +// The settings table always lives in the local SQLite database (config is +// per-operator, never on the shared MySQL logbook), so SQLite syntax is used +// unconditionally. The backticks around `key` are accepted by SQLite too. func (s *Store) SetRaw(ctx context.Context, key, value string) error { - _, err := s.db.ExecContext(ctx, ` - INSERT INTO settings(key, value) VALUES(?, ?) - ON CONFLICT(key) DO UPDATE SET - value = excluded.value, - updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`, - key, value) - if err != nil { + q := "INSERT INTO settings(`key`, value, updated_at) VALUES(?, ?, ?) " + + "ON CONFLICT(`key`) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at" + if _, err := s.db.ExecContext(ctx, q, key, value, db.NowISO()); err != nil { return fmt.Errorf("set %s: %w", key, err) } + s.cachePut(key, value) return nil } @@ -144,31 +194,41 @@ func (s *Store) Set(ctx context.Context, key, value string) error { // All returns every stored setting (sensitive values decrypted when unlocked). func (s *Store) All(ctx context.Context) (map[string]string, error) { - rows, err := s.db.QueryContext(ctx, `SELECT key, value FROM settings`) - if err != nil { + if err := s.ensureCache(ctx); err != nil { return nil, err } - defer rows.Close() - out := map[string]string{} - for rows.Next() { - var k, v string - if err := rows.Scan(&k, &v); err != nil { - return nil, err - } - out[k] = s.decodeRead(k, v) + s.mu.RLock() + raw := make(map[string]string, len(s.cache)) + for k, v := range s.cache { + raw[k] = v } - return out, rows.Err() -} - -// GetMany fetches several keys in a single round-trip. -func (s *Store) GetMany(ctx context.Context, keys ...string) (map[string]string, error) { - out := make(map[string]string, len(keys)) - for _, k := range keys { - v, err := s.Get(ctx, k) - if err != nil { - return nil, err - } - out[k] = v + s.mu.RUnlock() + out := make(map[string]string, len(raw)) + for k, v := range raw { + out[k] = s.decodeRead(k, v) + } + return out, nil +} + +// GetMany fetches several keys, all served from the in-memory cache (one DB +// round-trip total, on first access). Every requested key is present in the +// result (absent settings map to ""). +func (s *Store) GetMany(ctx context.Context, keys ...string) (map[string]string, error) { + out := make(map[string]string, len(keys)) + if len(keys) == 0 { + return out, nil + } + if err := s.ensureCache(ctx); err != nil { + return nil, err + } + s.mu.RLock() + raw := make([]string, len(keys)) + for i, k := range keys { + raw[i] = s.cache[k] + } + s.mu.RUnlock() + for i, k := range keys { + out[k] = s.decodeRead(k, raw[i]) } return out, nil }