// Package syncfolder keeps one operator's logbook in step across several PCs // through a folder they already have — Seafile, OneDrive, Dropbox, a NAS share, // a USB stick. // // WHY NOT THE DATABASE FILE ITSELF. Because it corrupts. SQLite relies on // advisory file locks that SMB and NFS implement partially or cache, so two // machines can both believe they hold the lock; and in WAL mode — which OpsLog // uses — the shared-memory index (-shm) has no meaning across machines at all. // A cloud folder is worse again: it replicates the file whole while it is open, // and .db / .db-wal / .db-shm drift apart, giving a database that opens // perfectly and is silently missing the last few hours. // // THE RULE THAT MAKES A SHARED FOLDER SAFE: one writer per file, append only. // Each machine writes ONLY its own .ndjson and never touches another's. // A sync tool that replicates whole files can therefore never merge two writers // into one file, because there are never two writers. This is the exact opposite // of putting the database there, and it is why it works. // // WHAT THIS IS NOT. Not live. Two operators logging the same contest second by // second want the shared MySQL logbook, which OpsLog already does; that is a // different need and it stays. This is for ONE operator with a shack PC, a // laptop and a portable rig — the case where a server running day and night to // serve forty QSOs a month is the wrong shape. // // It is the other half of internal/offlineq, whose own doc says it is // "deliberately NOT a sync engine: no mirror, no pull, no merge, no tombstones". // Those four are precisely what is here. package syncfolder import ( "crypto/rand" "encoding/hex" "encoding/json" "fmt" "os" "path/filepath" "strings" "time" ) // DirName is the sub-folder created inside whatever the operator picked. A // dedicated folder, so pointing OpsLog at a documents directory by mistake does // not scatter files through it. const DirName = "opslog-sync" // FormatVersion is stamped on every record. A future OpsLog that changes the // shape can then recognise — and skip — what it does not understand, instead of // misreading it. Records from the future are skipped, never guessed at. const FormatVersion = 1 // Op is what happened to a contact. type Op string const ( OpAdd Op = "add" OpUpdate Op = "update" // OpDelete is a TOMBSTONE, and it is the reason this is a change log rather // than a pile of ADIF. Without one, a QSO deleted on the laptop comes // straight back on the next sync from the shack PC, for ever. OpDelete Op = "delete" ) // Record is one line of a machine's file. NDJSON: one object per line, appended, // never rewritten — so a half-written line at the tail costs one record, not the // file, and a reader can resume from a byte offset. type Record struct { V int `json:"v"` Op Op `json:"op"` UID string `json:"uid"` // the contact's stable identity At time.Time `json:"at"` // when this CHANGE was made, UTC Seq uint64 `json:"seq"` // this machine's own counter, monotonic By string `json:"by"` // machine id that wrote it Data json.RawMessage `json:"data,omitempty"` } // NewUID mints a contact's identity. // // A stable id per contact is what lets an edit or a deletion be addressed at // all: "the QSO with M0ABC at 14:32" is a guess, and two machines can disagree // about which row that is. OpsLog already mints one for the offline outbox // (APP_OPSLOG_QUEUEID); this is the same idea, kept for the life of the record. func NewUID() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { return fmt.Sprintf("t%d", time.Now().UnixNano()) } return hex.EncodeToString(b[:]) } // NewMachineID mints this installation's id, once. The name is the operator's // (they may call it "shack" or "portable"); the suffix keeps two machines named // the same from writing to one file. func NewMachineID(name string) string { var b [4]byte _, _ = rand.Read(b[:]) n := sanitiseName(name) if n == "" { n = "opslog" } return n + "-" + hex.EncodeToString(b[:]) } // sanitiseName keeps a machine id usable as a FILENAME on every platform the // folder may be synced across — a Windows name written to a Linux NAS and back. func sanitiseName(s string) string { s = strings.ToLower(strings.TrimSpace(s)) var b strings.Builder for _, r := range s { switch { case r >= 'a' && r <= 'z', r >= '0' && r <= '9': b.WriteRune(r) case r == '-' || r == '_' || r == ' ': b.WriteByte('-') } } out := strings.Trim(b.String(), "-") for strings.Contains(out, "--") { out = strings.ReplaceAll(out, "--", "-") } if len(out) > 24 { out = strings.Trim(out[:24], "-") } return out } // Wins decides between two changes to the same contact. // // Last writer wins, but ordered on (At, Seq, By) rather than on time alone. // Two PCs' clocks are never exactly equal and one may be minutes out, so a bare // timestamp comparison is not even deterministic: two machines merging the same // pair in different orders could reach different answers and then disagree for // ever. Adding the writer's own counter and finally its id makes the order // total — every machine reaches the same conclusion from the same records, // whatever sequence they arrive in. // // Clock skew still decides WHICH edit wins, and nothing here can fix that. What // it guarantees is that all machines agree on the winner. func Wins(a, b Record) bool { if !a.At.Equal(b.At) { return a.At.After(b.At) } if a.Seq != b.Seq { return a.Seq > b.Seq } return a.By > b.By } // Store is one machine's view of the shared folder. type Store struct { root string // the folder the operator picked machineID string } // New returns a store. root is the operator's chosen folder; the package // creates and uses its own sub-folder inside it. func New(root, machineID string) *Store { return &Store{root: strings.TrimSpace(root), machineID: machineID} } // Dir is where the files live. func (s *Store) Dir() string { return filepath.Join(s.root, DirName) } // MyFile is the only file this machine ever writes. func (s *Store) MyFile() string { return filepath.Join(s.Dir(), s.machineID+".ndjson") } // Append adds one record to this machine's file. // // Opened, written and closed per call, with O_APPEND: a sync client that // uploads the file between two contacts sees a complete file every time, and a // process killed mid-write loses at most the line it was writing. func (s *Store) Append(rec Record) error { if s.root == "" { return fmt.Errorf("syncfolder: no folder configured") } if err := os.MkdirAll(s.Dir(), 0o755); err != nil { return fmt.Errorf("syncfolder: create %s: %w", s.Dir(), err) } rec.V = FormatVersion rec.By = s.machineID if rec.At.IsZero() { rec.At = time.Now().UTC() } rec.At = rec.At.UTC() line, err := json.Marshal(rec) if err != nil { return fmt.Errorf("syncfolder: encode: %w", err) } f, err := os.OpenFile(s.MyFile(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return fmt.Errorf("syncfolder: open %s: %w", s.MyFile(), err) } defer f.Close() if _, err := f.Write(append(line, '\n')); err != nil { return fmt.Errorf("syncfolder: write: %w", err) } return nil } // Peer is another machine's file and how far this one has read it. type Peer struct { MachineID string Path string Size int64 } // Peers lists the other machines' files, skipping this machine's own. func (s *Store) Peers() ([]Peer, error) { if s.root == "" { return nil, fmt.Errorf("syncfolder: no folder configured") } entries, err := os.ReadDir(s.Dir()) if err != nil { if os.IsNotExist(err) { return nil, nil // nobody has written anything yet } return nil, err } var out []Peer for _, e := range entries { if e.IsDir() || !strings.HasSuffix(e.Name(), ".ndjson") { continue } id := strings.TrimSuffix(e.Name(), ".ndjson") if id == s.machineID { continue // never read our own back — that is how a loop starts } info, err := e.Info() if err != nil { continue } out = append(out, Peer{MachineID: id, Path: filepath.Join(s.Dir(), e.Name()), Size: info.Size()}) } return out, nil } // ReadFrom returns the records in a peer's file after byte offset `from`, and // the offset to resume at next time. // // Resuming by byte offset is what keeps a sync cheap: a file with 40 000 // contacts is read once, and thereafter only its tail. The returned offset // advances ONLY past complete lines — a file caught mid-upload ends in a // partial line, and stopping short of it means the next pass reads that record // whole instead of discarding it. func ReadFrom(path string, from int64) ([]Record, int64, error) { data, err := os.ReadFile(path) if err != nil { return nil, from, err } // A file that SHRANK was replaced, not appended to — a sync conflict copy, // a restore, a machine id reused. Start again rather than read from an // offset that now points into the middle of a different record. if int64(len(data)) < from { from = 0 } tail := data[from:] var recs []Record consumed := int64(0) for { i := indexByte(tail, '\n') if i < 0 { break // an incomplete final line: leave it for next time } line := tail[:i] tail = tail[i+1:] consumed += int64(i) + 1 if len(strings.TrimSpace(string(line))) == 0 { continue } var rec Record if err := json.Unmarshal(line, &rec); err != nil { continue // one unreadable line must not stop the file } if rec.V > FormatVersion { continue // written by a newer OpsLog: skip, never guess } if rec.UID == "" || rec.Op == "" { continue } recs = append(recs, rec) } return recs, from + consumed, nil } func indexByte(b []byte, c byte) int { for i := range b { if b[i] == c { return i } } return -1 } // Merge reduces a batch of records to ONE decision per contact — the winner. // // Applying every record in turn would work but would write the same row several // times over, and on a first sync of a large log that is thousands of pointless // updates. It also makes the result independent of the order the peers' // files happened to be read in. func Merge(recs []Record) map[string]Record { out := make(map[string]Record, len(recs)) for _, r := range recs { cur, seen := out[r.UID] if !seen || Wins(r, cur) { out[r.UID] = r } } return out }