From 7d664bd1defd289fdbc83d482e6f81c1736839cd Mon Sep 17 00:00:00 2001 From: Gregory Salaun Date: Sun, 16 Aug 2026 21:44:14 +0200 Subject: [PATCH] feat(sync): the change-log core for one operator on several PCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shape we agreed: each PC keeps its own local SQLite and they exchange CHANGES through a folder the operator already has — Seafile, OneDrive, a NAS, a USB stick. Not the database file: SQLite over SMB or NFS corrupts, and in WAL mode the shared-memory index has no meaning across machines at all. The rule that makes a shared folder safe is one writer per file, append only. Each machine writes only its own .ndjson and never touches another's, so a sync client that replicates whole files can never merge two writers into one — there are never two. That is the exact opposite of putting the database there, and it is why it works. This is the half internal/offlineq deliberately refuses to be: its own doc says "no mirror, no pull, no merge, no tombstones". All four are here. Four decisions worth naming, each with a test: Tombstones. A delete is a record. Without one, a QSO removed on the laptop comes back on the next sync from the shack PC, for ever — and a contact logged again after a deletion has to come back, which the ordering also has to allow. Determinism. Last writer wins, ordered on (time, the writer's own counter, the writer's id) — not on time alone. Two PCs' clocks are never equal, so a bare timestamp is not even a total order: two machines merging the same pair in different orders could reach different answers and disagree for ever. Skew still decides WHICH edit wins and nothing can fix that; what this guarantees is that every machine agrees on the winner. Resumption. Readers resume at a byte offset, so a 40 000-contact file is read once and thereafter only its tail — and the offset advances only past COMPLETE lines, because a folder sync catches a file mid-upload sooner or later. A file that SHRANK was replaced rather than appended to, and is re-read from the start. Survivability. One unreadable line costs one record. A record stamped with a FUTURE format version is skipped, never guessed at. Core only: no UI and nothing wired to the logbook yet, so nothing is user-visible and there is no changelog entry. Next is the sync_uid column, the add/update/delete hooks and the settings panel. --- internal/syncfolder/syncfolder.go | 308 +++++++++++++++++++++++++ internal/syncfolder/syncfolder_test.go | 172 ++++++++++++++ 2 files changed, 480 insertions(+) create mode 100644 internal/syncfolder/syncfolder.go create mode 100644 internal/syncfolder/syncfolder_test.go diff --git a/internal/syncfolder/syncfolder.go b/internal/syncfolder/syncfolder.go new file mode 100644 index 0000000..3534b84 --- /dev/null +++ b/internal/syncfolder/syncfolder.go @@ -0,0 +1,308 @@ +// Package syncfolder keeps one operator's logbook in step across several PCs +// through a folder they already have — Seafile, OneDrive, Dropbox, a NAS share, +// a USB stick. +// +// WHY NOT THE DATABASE FILE ITSELF. Because it corrupts. SQLite relies on +// advisory file locks that SMB and NFS implement partially or cache, so two +// machines can both believe they hold the lock; and in WAL mode — which OpsLog +// uses — the shared-memory index (-shm) has no meaning across machines at all. +// A cloud folder is worse again: it replicates the file whole while it is open, +// and .db / .db-wal / .db-shm drift apart, giving a database that opens +// perfectly and is silently missing the last few hours. +// +// THE RULE THAT MAKES A SHARED FOLDER SAFE: one writer per file, append only. +// Each machine writes ONLY its own .ndjson and never touches another's. +// A sync tool that replicates whole files can therefore never merge two writers +// into one file, because there are never two writers. This is the exact opposite +// of putting the database there, and it is why it works. +// +// WHAT THIS IS NOT. Not live. Two operators logging the same contest second by +// second want the shared MySQL logbook, which OpsLog already does; that is a +// different need and it stays. This is for ONE operator with a shack PC, a +// laptop and a portable rig — the case where a server running day and night to +// serve forty QSOs a month is the wrong shape. +// +// It is the other half of internal/offlineq, whose own doc says it is +// "deliberately NOT a sync engine: no mirror, no pull, no merge, no tombstones". +// Those four are precisely what is here. +package syncfolder + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// DirName is the sub-folder created inside whatever the operator picked. A +// dedicated folder, so pointing OpsLog at a documents directory by mistake does +// not scatter files through it. +const DirName = "opslog-sync" + +// FormatVersion is stamped on every record. A future OpsLog that changes the +// shape can then recognise — and skip — what it does not understand, instead of +// misreading it. Records from the future are skipped, never guessed at. +const FormatVersion = 1 + +// Op is what happened to a contact. +type Op string + +const ( + OpAdd Op = "add" + OpUpdate Op = "update" + // OpDelete is a TOMBSTONE, and it is the reason this is a change log rather + // than a pile of ADIF. Without one, a QSO deleted on the laptop comes + // straight back on the next sync from the shack PC, for ever. + OpDelete Op = "delete" +) + +// Record is one line of a machine's file. NDJSON: one object per line, appended, +// never rewritten — so a half-written line at the tail costs one record, not the +// file, and a reader can resume from a byte offset. +type Record struct { + V int `json:"v"` + Op Op `json:"op"` + UID string `json:"uid"` // the contact's stable identity + At time.Time `json:"at"` // when this CHANGE was made, UTC + Seq uint64 `json:"seq"` // this machine's own counter, monotonic + By string `json:"by"` // machine id that wrote it + Data json.RawMessage `json:"data,omitempty"` +} + +// NewUID mints a contact's identity. +// +// A stable id per contact is what lets an edit or a deletion be addressed at +// all: "the QSO with M0ABC at 14:32" is a guess, and two machines can disagree +// about which row that is. OpsLog already mints one for the offline outbox +// (APP_OPSLOG_QUEUEID); this is the same idea, kept for the life of the record. +func NewUID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("t%d", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} + +// NewMachineID mints this installation's id, once. The name is the operator's +// (they may call it "shack" or "portable"); the suffix keeps two machines named +// the same from writing to one file. +func NewMachineID(name string) string { + var b [4]byte + _, _ = rand.Read(b[:]) + n := sanitiseName(name) + if n == "" { + n = "opslog" + } + return n + "-" + hex.EncodeToString(b[:]) +} + +// sanitiseName keeps a machine id usable as a FILENAME on every platform the +// folder may be synced across — a Windows name written to a Linux NAS and back. +func sanitiseName(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-' || r == '_' || r == ' ': + b.WriteByte('-') + } + } + out := strings.Trim(b.String(), "-") + for strings.Contains(out, "--") { + out = strings.ReplaceAll(out, "--", "-") + } + if len(out) > 24 { + out = strings.Trim(out[:24], "-") + } + return out +} + +// Wins decides between two changes to the same contact. +// +// Last writer wins, but ordered on (At, Seq, By) rather than on time alone. +// Two PCs' clocks are never exactly equal and one may be minutes out, so a bare +// timestamp comparison is not even deterministic: two machines merging the same +// pair in different orders could reach different answers and then disagree for +// ever. Adding the writer's own counter and finally its id makes the order +// total — every machine reaches the same conclusion from the same records, +// whatever sequence they arrive in. +// +// Clock skew still decides WHICH edit wins, and nothing here can fix that. What +// it guarantees is that all machines agree on the winner. +func Wins(a, b Record) bool { + if !a.At.Equal(b.At) { + return a.At.After(b.At) + } + if a.Seq != b.Seq { + return a.Seq > b.Seq + } + return a.By > b.By +} + +// Store is one machine's view of the shared folder. +type Store struct { + root string // the folder the operator picked + machineID string +} + +// New returns a store. root is the operator's chosen folder; the package +// creates and uses its own sub-folder inside it. +func New(root, machineID string) *Store { + return &Store{root: strings.TrimSpace(root), machineID: machineID} +} + +// Dir is where the files live. +func (s *Store) Dir() string { return filepath.Join(s.root, DirName) } + +// MyFile is the only file this machine ever writes. +func (s *Store) MyFile() string { return filepath.Join(s.Dir(), s.machineID+".ndjson") } + +// Append adds one record to this machine's file. +// +// Opened, written and closed per call, with O_APPEND: a sync client that +// uploads the file between two contacts sees a complete file every time, and a +// process killed mid-write loses at most the line it was writing. +func (s *Store) Append(rec Record) error { + if s.root == "" { + return fmt.Errorf("syncfolder: no folder configured") + } + if err := os.MkdirAll(s.Dir(), 0o755); err != nil { + return fmt.Errorf("syncfolder: create %s: %w", s.Dir(), err) + } + rec.V = FormatVersion + rec.By = s.machineID + if rec.At.IsZero() { + rec.At = time.Now().UTC() + } + rec.At = rec.At.UTC() + line, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("syncfolder: encode: %w", err) + } + f, err := os.OpenFile(s.MyFile(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("syncfolder: open %s: %w", s.MyFile(), err) + } + defer f.Close() + if _, err := f.Write(append(line, '\n')); err != nil { + return fmt.Errorf("syncfolder: write: %w", err) + } + return nil +} + +// Peer is another machine's file and how far this one has read it. +type Peer struct { + MachineID string + Path string + Size int64 +} + +// Peers lists the other machines' files, skipping this machine's own. +func (s *Store) Peers() ([]Peer, error) { + if s.root == "" { + return nil, fmt.Errorf("syncfolder: no folder configured") + } + entries, err := os.ReadDir(s.Dir()) + if err != nil { + if os.IsNotExist(err) { + return nil, nil // nobody has written anything yet + } + return nil, err + } + var out []Peer + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".ndjson") { + continue + } + id := strings.TrimSuffix(e.Name(), ".ndjson") + if id == s.machineID { + continue // never read our own back — that is how a loop starts + } + info, err := e.Info() + if err != nil { + continue + } + out = append(out, Peer{MachineID: id, Path: filepath.Join(s.Dir(), e.Name()), Size: info.Size()}) + } + return out, nil +} + +// ReadFrom returns the records in a peer's file after byte offset `from`, and +// the offset to resume at next time. +// +// Resuming by byte offset is what keeps a sync cheap: a file with 40 000 +// contacts is read once, and thereafter only its tail. The returned offset +// advances ONLY past complete lines — a file caught mid-upload ends in a +// partial line, and stopping short of it means the next pass reads that record +// whole instead of discarding it. +func ReadFrom(path string, from int64) ([]Record, int64, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, from, err + } + // A file that SHRANK was replaced, not appended to — a sync conflict copy, + // a restore, a machine id reused. Start again rather than read from an + // offset that now points into the middle of a different record. + if int64(len(data)) < from { + from = 0 + } + tail := data[from:] + var recs []Record + consumed := int64(0) + for { + i := indexByte(tail, '\n') + if i < 0 { + break // an incomplete final line: leave it for next time + } + line := tail[:i] + tail = tail[i+1:] + consumed += int64(i) + 1 + if len(strings.TrimSpace(string(line))) == 0 { + continue + } + var rec Record + if err := json.Unmarshal(line, &rec); err != nil { + continue // one unreadable line must not stop the file + } + if rec.V > FormatVersion { + continue // written by a newer OpsLog: skip, never guess + } + if rec.UID == "" || rec.Op == "" { + continue + } + recs = append(recs, rec) + } + return recs, from + consumed, nil +} + +func indexByte(b []byte, c byte) int { + for i := range b { + if b[i] == c { + return i + } + } + return -1 +} + +// Merge reduces a batch of records to ONE decision per contact — the winner. +// +// Applying every record in turn would work but would write the same row several +// times over, and on a first sync of a large log that is thousands of pointless +// updates. It also makes the result independent of the order the peers' +// files happened to be read in. +func Merge(recs []Record) map[string]Record { + out := make(map[string]Record, len(recs)) + for _, r := range recs { + cur, seen := out[r.UID] + if !seen || Wins(r, cur) { + out[r.UID] = r + } + } + return out +} diff --git a/internal/syncfolder/syncfolder_test.go b/internal/syncfolder/syncfolder_test.go new file mode 100644 index 0000000..a168522 --- /dev/null +++ b/internal/syncfolder/syncfolder_test.go @@ -0,0 +1,172 @@ +package syncfolder + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func rec(uid string, op Op, at string, seq uint64, by string) Record { + t, _ := time.Parse(time.RFC3339, at) + return Record{V: FormatVersion, Op: op, UID: uid, At: t, Seq: seq, By: by} +} + +// A machine must never read its own file back. That is how a change loops +// round the folder for ever, each pass re-applying what this machine wrote. +func TestPeersExcludesOurselves(t *testing.T) { + root := t.TempDir() + s := New(root, "shack-aabbccdd") + if err := s.Append(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "")); err != nil { + t.Fatalf("append: %v", err) + } + must(t, os.WriteFile(filepath.Join(s.Dir(), "laptop-11223344.ndjson"), []byte("{}\n"), 0o644)) + + peers, err := s.Peers() + if err != nil { + t.Fatalf("peers: %v", err) + } + if len(peers) != 1 || peers[0].MachineID != "laptop-11223344" { + t.Fatalf("peers = %+v — our own file must not be among them", peers) + } +} + +// Reading resumes from a byte offset, and only ever advances past COMPLETE +// lines. A folder sync catches a file mid-upload sooner or later, and the +// partial last line must be read whole on the next pass, not thrown away. +func TestReadFromStopsAtAnIncompleteLine(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "peer.ndjson") + + full, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer")) + partial, _ := json.Marshal(rec("u2", OpAdd, "2026-08-16T10:01:00Z", 2, "peer")) + must(t, os.WriteFile(path, append(append(full, '\n'), partial[:len(partial)/2]...), 0o644)) + + recs, off, err := ReadFrom(path, 0) + if err != nil { + t.Fatalf("ReadFrom: %v", err) + } + if len(recs) != 1 || recs[0].UID != "u1" { + t.Fatalf("got %d record(s) %+v, want just u1", len(recs), recs) + } + if off != int64(len(full))+1 { + t.Fatalf("offset %d, want %d — it must stop before the partial line", off, len(full)+1) + } + + // The upload completes; the second record is now read whole. + must(t, os.WriteFile(path, append(append(append(full, '\n'), partial...), '\n'), 0o644)) + recs, off2, err := ReadFrom(path, off) + if err != nil { + t.Fatalf("ReadFrom 2: %v", err) + } + if len(recs) != 1 || recs[0].UID != "u2" { + t.Fatalf("resumed with %+v, want just u2", recs) + } + if off2 <= off { + t.Errorf("offset did not advance: %d → %d", off, off2) + } +} + +// A file that shrank was replaced, not appended to — a sync conflict copy, a +// restore. Resuming at the old offset would read from the middle of a record. +func TestReadFromRestartsIfTheFileShrank(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "peer.ndjson") + line, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer")) + must(t, os.WriteFile(path, append(line, '\n'), 0o644)) + + recs, _, err := ReadFrom(path, 999999) + if err != nil { + t.Fatalf("ReadFrom: %v", err) + } + if len(recs) != 1 { + t.Errorf("got %d record(s) — a shrunken file must be re-read from the start", len(recs)) + } +} + +// One unreadable line must not cost the rest of the file, and a record from a +// FUTURE format must be skipped rather than guessed at. +func TestReadFromSurvivesRubbish(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "peer.ndjson") + good, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer")) + future, _ := json.Marshal(Record{V: FormatVersion + 1, Op: OpAdd, UID: "u9", At: time.Now().UTC()}) + body := append([]byte("this is not json\n"), append(good, '\n')...) + body = append(body, append(future, '\n')...) + body = append(body, []byte("\n")...) // a blank line + must(t, os.WriteFile(path, body, 0o644)) + + recs, _, err := ReadFrom(path, 0) + if err != nil { + t.Fatalf("ReadFrom: %v", err) + } + if len(recs) != 1 || recs[0].UID != "u1" { + t.Fatalf("got %+v, want only the one readable current-format record", recs) + } +} + +// Every machine must reach the SAME winner from the same records, whatever +// order they arrive in. Ordering on time alone is not even deterministic: two +// PCs' clocks are never equal, and a tie would be resolved differently on each +// machine, which is how two logs disagree for ever. +func TestMergeIsOrderIndependent(t *testing.T) { + a := rec("u1", OpUpdate, "2026-08-16T10:00:00Z", 5, "shack") + b := rec("u1", OpDelete, "2026-08-16T10:00:00Z", 5, "laptop") // same time AND seq + c := rec("u1", OpUpdate, "2026-08-16T09:00:00Z", 9, "shack") // older, higher seq + + forward := Merge([]Record{a, b, c}) + reverse := Merge([]Record{c, b, a}) + if forward["u1"].By != reverse["u1"].By || forward["u1"].Op != reverse["u1"].Op { + t.Fatalf("order changed the winner: %+v vs %+v", forward["u1"], reverse["u1"]) + } + // Time beats sequence: c is an hour older whatever its counter says. + if forward["u1"].At.Equal(c.At) { + t.Error("an older change won on its counter — time is the first key") + } +} + +// A delete is a record like any other, and it must be able to WIN. Without +// tombstones a QSO removed on one machine returns on the next sync from +// another, for ever. +func TestATombstoneCanWin(t *testing.T) { + add := rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "shack") + del := rec("u1", OpDelete, "2026-08-16T11:00:00Z", 1, "laptop") + if got := Merge([]Record{add, del})["u1"]; got.Op != OpDelete { + t.Errorf("winner is %q — a later deletion must beat an earlier add", got.Op) + } + // …and an add made AFTER a deletion wins, so re-logging a contact works. + readd := rec("u1", OpAdd, "2026-08-16T12:00:00Z", 2, "laptop") + if got := Merge([]Record{add, del, readd})["u1"]; got.Op != OpAdd { + t.Errorf("winner is %q — a contact logged again after a deletion must come back", got.Op) + } +} + +// The machine id becomes a FILENAME, on a folder that may be synced between +// Windows, Linux and macOS. Anything that cannot be a filename everywhere has +// to go. +func TestMachineIDIsAUsableFilename(t *testing.T) { + for _, name := range []string{"Shack PC", "portable/rig", "Café ☕", "", " ", "a::b*c?", strings.Repeat("x", 60)} { + id := NewMachineID(name) + for _, bad := range []string{"/", "\\", ":", "*", "?", "\"", "<", ">", "|", " "} { + if strings.Contains(id, bad) { + t.Errorf("NewMachineID(%q) = %q contains %q", name, id, bad) + } + } + if id == "" || strings.HasPrefix(id, "-") || strings.HasSuffix(id, "-") { + t.Errorf("NewMachineID(%q) = %q", name, id) + } + } + // Two machines the operator called the same must not share a file. + if NewMachineID("shack") == NewMachineID("shack") { + t.Error("two installations named alike produced the same id — they would write to one file") + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +}