OpsLog/internal/syncfolder/syncfolder.go

// Package syncfolder keeps one operator's logbook in step across several PCs
// through a folder they already have — Seafile, OneDrive, Dropbox, a NAS share,
// a USB stick.
//
// WHY NOT THE DATABASE FILE ITSELF. Because it corrupts. SQLite relies on
// advisory file locks that SMB and NFS implement partially or cache, so two
// machines can both believe they hold the lock; and in WAL mode — which OpsLog
// uses — the shared-memory index (-shm) has no meaning across machines at all.
// A cloud folder is worse again: it replicates the file whole while it is open,
// and .db / .db-wal / .db-shm drift apart, giving a database that opens
// perfectly and is silently missing the last few hours.
//
// THE RULE THAT MAKES A SHARED FOLDER SAFE: one writer per file, append only.
// Each machine writes ONLY its own <machine>.ndjson and never touches another's.
// A sync tool that replicates whole files can therefore never merge two writers
// into one file, because there are never two writers. This is the exact opposite
// of putting the database there, and it is why it works.
//
// WHAT THIS IS NOT. Not live. Two operators logging the same contest second by
// second want the shared MySQL logbook, which OpsLog already does; that is a
// different need and it stays. This is for ONE operator with a shack PC, a
// laptop and a portable rig — the case where a server running day and night to
// serve forty QSOs a month is the wrong shape.
//
// It is the other half of internal/offlineq, whose own doc says it is
// "deliberately NOT a sync engine: no mirror, no pull, no merge, no tombstones".
// Those four are precisely what is here.
package syncfolder

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// DirName is the sub-folder created inside whatever the operator picked. A
// dedicated folder, so pointing OpsLog at a documents directory by mistake does
// not scatter files through it.
const DirName = "opslog-sync"

// FormatVersion is stamped on every record. A future OpsLog that changes the
// shape can then recognise — and skip — what it does not understand, instead of
// misreading it. Records from the future are skipped, never guessed at.
const FormatVersion = 1

// Op is what happened to a contact.
type Op string

const (
	OpAdd    Op = "add"
	OpUpdate Op = "update"
	// OpDelete is a TOMBSTONE, and it is the reason this is a change log rather
	// than a pile of ADIF. Without one, a QSO deleted on the laptop comes
	// straight back on the next sync from the shack PC, for ever.
	OpDelete Op = "delete"
)

// Record is one line of a machine's file. NDJSON: one object per line, appended,
// never rewritten — so a half-written line at the tail costs one record, not the
// file, and a reader can resume from a byte offset.
type Record struct {
	V    int             `json:"v"`
	Op   Op              `json:"op"`
	UID  string          `json:"uid"` // the contact's stable identity
	At   time.Time       `json:"at"`  // when this CHANGE was made, UTC
	Seq  uint64          `json:"seq"` // this machine's own counter, monotonic
	By   string          `json:"by"`  // machine id that wrote it
	Data json.RawMessage `json:"data,omitempty"`
}

// NewUID mints a contact's identity.
//
// A stable id per contact is what lets an edit or a deletion be addressed at
// all: "the QSO with M0ABC at 14:32" is a guess, and two machines can disagree
// about which row that is. OpsLog already mints one for the offline outbox
// (APP_OPSLOG_QUEUEID); this is the same idea, kept for the life of the record.
func NewUID() string {
	var b [16]byte
	if _, err := rand.Read(b[:]); err != nil {
		return fmt.Sprintf("t%d", time.Now().UnixNano())
	}
	return hex.EncodeToString(b[:])
}

// NewMachineID mints this installation's id, once. The name is the operator's
// (they may call it "shack" or "portable"); the suffix keeps two machines named
// the same from writing to one file.
func NewMachineID(name string) string {
	var b [4]byte
	_, _ = rand.Read(b[:])
	n := sanitiseName(name)
	if n == "" {
		n = "opslog"
	}
	return n + "-" + hex.EncodeToString(b[:])
}

// sanitiseName keeps a machine id usable as a FILENAME on every platform the
// folder may be synced across — a Windows name written to a Linux NAS and back.
func sanitiseName(s string) string {
	s = strings.ToLower(strings.TrimSpace(s))
	var b strings.Builder
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
		case r == '-' || r == '_' || r == ' ':
			b.WriteByte('-')
		}
	}
	out := strings.Trim(b.String(), "-")
	for strings.Contains(out, "--") {
		out = strings.ReplaceAll(out, "--", "-")
	}
	if len(out) > 24 {
		out = strings.Trim(out[:24], "-")
	}
	return out
}

// Wins decides between two changes to the same contact.
//
// Last writer wins, but ordered on (At, Seq, By) rather than on time alone.
// Two PCs' clocks are never exactly equal and one may be minutes out, so a bare
// timestamp comparison is not even deterministic: two machines merging the same
// pair in different orders could reach different answers and then disagree for
// ever. Adding the writer's own counter and finally its id makes the order
// total — every machine reaches the same conclusion from the same records,
// whatever sequence they arrive in.
//
// Clock skew still decides WHICH edit wins, and nothing here can fix that. What
// it guarantees is that all machines agree on the winner.
func Wins(a, b Record) bool {
	if !a.At.Equal(b.At) {
		return a.At.After(b.At)
	}
	if a.Seq != b.Seq {
		return a.Seq > b.Seq
	}
	return a.By > b.By
}

// Store is one machine's view of the shared folder.
type Store struct {
	root      string // the folder the operator picked
	machineID string
}

// New returns a store. root is the operator's chosen folder; the package
// creates and uses its own sub-folder inside it.
func New(root, machineID string) *Store {
	return &Store{root: strings.TrimSpace(root), machineID: machineID}
}

// Dir is where the files live.
func (s *Store) Dir() string { return filepath.Join(s.root, DirName) }

// MyFile is the only file this machine ever writes.
func (s *Store) MyFile() string { return filepath.Join(s.Dir(), s.machineID+".ndjson") }

// Append adds one record to this machine's file.
//
// Opened, written and closed per call, with O_APPEND: a sync client that
// uploads the file between two contacts sees a complete file every time, and a
// process killed mid-write loses at most the line it was writing.
func (s *Store) Append(rec Record) error {
	if s.root == "" {
		return fmt.Errorf("syncfolder: no folder configured")
	}
	if err := os.MkdirAll(s.Dir(), 0o755); err != nil {
		return fmt.Errorf("syncfolder: create %s: %w", s.Dir(), err)
	}
	rec.V = FormatVersion
	rec.By = s.machineID
	if rec.At.IsZero() {
		rec.At = time.Now().UTC()
	}
	rec.At = rec.At.UTC()
	line, err := json.Marshal(rec)
	if err != nil {
		return fmt.Errorf("syncfolder: encode: %w", err)
	}
	f, err := os.OpenFile(s.MyFile(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		return fmt.Errorf("syncfolder: open %s: %w", s.MyFile(), err)
	}
	defer f.Close()
	if _, err := f.Write(append(line, '\n')); err != nil {
		return fmt.Errorf("syncfolder: write: %w", err)
	}
	return nil
}

// Peer is another machine's file and how far this one has read it.
type Peer struct {
	MachineID string
	Path      string
	Size      int64
}

// Peers lists the other machines' files, skipping this machine's own.
func (s *Store) Peers() ([]Peer, error) {
	if s.root == "" {
		return nil, fmt.Errorf("syncfolder: no folder configured")
	}
	entries, err := os.ReadDir(s.Dir())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil // nobody has written anything yet
		}
		return nil, err
	}
	var out []Peer
	for _, e := range entries {
		if e.IsDir() || !strings.HasSuffix(e.Name(), ".ndjson") {
			continue
		}
		id := strings.TrimSuffix(e.Name(), ".ndjson")
		if id == s.machineID {
			continue // never read our own back — that is how a loop starts
		}
		info, err := e.Info()
		if err != nil {
			continue
		}
		out = append(out, Peer{MachineID: id, Path: filepath.Join(s.Dir(), e.Name()), Size: info.Size()})
	}
	return out, nil
}

// ReadFrom returns the records in a peer's file after byte offset `from`, and
// the offset to resume at next time.
//
// Resuming by byte offset is what keeps a sync cheap: a file with 40 000
// contacts is read once, and thereafter only its tail. The returned offset
// advances ONLY past complete lines — a file caught mid-upload ends in a
// partial line, and stopping short of it means the next pass reads that record
// whole instead of discarding it.
func ReadFrom(path string, from int64) ([]Record, int64, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, from, err
	}
	// A file that SHRANK was replaced, not appended to — a sync conflict copy,
	// a restore, a machine id reused. Start again rather than read from an
	// offset that now points into the middle of a different record.
	if int64(len(data)) < from {
		from = 0
	}
	tail := data[from:]
	var recs []Record
	consumed := int64(0)
	for {
		i := indexByte(tail, '\n')
		if i < 0 {
			break // an incomplete final line: leave it for next time
		}
		line := tail[:i]
		tail = tail[i+1:]
		consumed += int64(i) + 1
		if len(strings.TrimSpace(string(line))) == 0 {
			continue
		}
		var rec Record
		if err := json.Unmarshal(line, &rec); err != nil {
			continue // one unreadable line must not stop the file
		}
		if rec.V > FormatVersion {
			continue // written by a newer OpsLog: skip, never guess
		}
		if rec.UID == "" || rec.Op == "" {
			continue
		}
		recs = append(recs, rec)
	}
	return recs, from + consumed, nil
}

func indexByte(b []byte, c byte) int {
	for i := range b {
		if b[i] == c {
			return i
		}
	}
	return -1
}

// Merge reduces a batch of records to ONE decision per contact — the winner.
//
// Applying every record in turn would work but would write the same row several
// times over, and on a first sync of a large log that is thousands of pointless
// updates. It also makes the result independent of the order the peers'
// files happened to be read in.
func Merge(recs []Record) map[string]Record {
	out := make(map[string]Record, len(recs))
	for _, r := range recs {
		cur, seen := out[r.UID]
		if !seen || Wins(r, cur) {
			out[r.UID] = r
		}
	}
	return out
}