feat(sync): the change-log core for one operator on several PCs
The shape we agreed: each PC keeps its own local SQLite and they exchange CHANGES through a folder the operator already has — Seafile, OneDrive, a NAS, a USB stick. Not the database file: SQLite over SMB or NFS corrupts, and in WAL mode the shared-memory index has no meaning across machines at all. The rule that makes a shared folder safe is one writer per file, append only. Each machine writes only its own <machine>.ndjson and never touches another's, so a sync client that replicates whole files can never merge two writers into one — there are never two. That is the exact opposite of putting the database there, and it is why it works. This is the half internal/offlineq deliberately refuses to be: its own doc says "no mirror, no pull, no merge, no tombstones". All four are here. Four decisions worth naming, each with a test: Tombstones. A delete is a record. Without one, a QSO removed on the laptop comes back on the next sync from the shack PC, for ever — and a contact logged again after a deletion has to come back, which the ordering also has to allow. Determinism. Last writer wins, ordered on (time, the writer's own counter, the writer's id) — not on time alone. Two PCs' clocks are never equal, so a bare timestamp is not even a total order: two machines merging the same pair in different orders could reach different answers and disagree for ever. Skew still decides WHICH edit wins and nothing can fix that; what this guarantees is that every machine agrees on the winner. Resumption. Readers resume at a byte offset, so a 40 000-contact file is read once and thereafter only its tail — and the offset advances only past COMPLETE lines, because a folder sync catches a file mid-upload sooner or later. A file that SHRANK was replaced rather than appended to, and is re-read from the start. Survivability. One unreadable line costs one record. A record stamped with a FUTURE format version is skipped, never guessed at. Core only: no UI and nothing wired to the logbook yet, so nothing is user-visible and there is no changelog entry. Next is the sync_uid column, the add/update/delete hooks and the settings panel.
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
package syncfolder
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func rec(uid string, op Op, at string, seq uint64, by string) Record {
|
||||
t, _ := time.Parse(time.RFC3339, at)
|
||||
return Record{V: FormatVersion, Op: op, UID: uid, At: t, Seq: seq, By: by}
|
||||
}
|
||||
|
||||
// A machine must never read its own file back. That is how a change loops
|
||||
// round the folder for ever, each pass re-applying what this machine wrote.
|
||||
func TestPeersExcludesOurselves(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
s := New(root, "shack-aabbccdd")
|
||||
if err := s.Append(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "")); err != nil {
|
||||
t.Fatalf("append: %v", err)
|
||||
}
|
||||
must(t, os.WriteFile(filepath.Join(s.Dir(), "laptop-11223344.ndjson"), []byte("{}\n"), 0o644))
|
||||
|
||||
peers, err := s.Peers()
|
||||
if err != nil {
|
||||
t.Fatalf("peers: %v", err)
|
||||
}
|
||||
if len(peers) != 1 || peers[0].MachineID != "laptop-11223344" {
|
||||
t.Fatalf("peers = %+v — our own file must not be among them", peers)
|
||||
}
|
||||
}
|
||||
|
||||
// Reading resumes from a byte offset, and only ever advances past COMPLETE
|
||||
// lines. A folder sync catches a file mid-upload sooner or later, and the
|
||||
// partial last line must be read whole on the next pass, not thrown away.
|
||||
func TestReadFromStopsAtAnIncompleteLine(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "peer.ndjson")
|
||||
|
||||
full, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer"))
|
||||
partial, _ := json.Marshal(rec("u2", OpAdd, "2026-08-16T10:01:00Z", 2, "peer"))
|
||||
must(t, os.WriteFile(path, append(append(full, '\n'), partial[:len(partial)/2]...), 0o644))
|
||||
|
||||
recs, off, err := ReadFrom(path, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if len(recs) != 1 || recs[0].UID != "u1" {
|
||||
t.Fatalf("got %d record(s) %+v, want just u1", len(recs), recs)
|
||||
}
|
||||
if off != int64(len(full))+1 {
|
||||
t.Fatalf("offset %d, want %d — it must stop before the partial line", off, len(full)+1)
|
||||
}
|
||||
|
||||
// The upload completes; the second record is now read whole.
|
||||
must(t, os.WriteFile(path, append(append(append(full, '\n'), partial...), '\n'), 0o644))
|
||||
recs, off2, err := ReadFrom(path, off)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom 2: %v", err)
|
||||
}
|
||||
if len(recs) != 1 || recs[0].UID != "u2" {
|
||||
t.Fatalf("resumed with %+v, want just u2", recs)
|
||||
}
|
||||
if off2 <= off {
|
||||
t.Errorf("offset did not advance: %d → %d", off, off2)
|
||||
}
|
||||
}
|
||||
|
||||
// A file that shrank was replaced, not appended to — a sync conflict copy, a
|
||||
// restore. Resuming at the old offset would read from the middle of a record.
|
||||
func TestReadFromRestartsIfTheFileShrank(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "peer.ndjson")
|
||||
line, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer"))
|
||||
must(t, os.WriteFile(path, append(line, '\n'), 0o644))
|
||||
|
||||
recs, _, err := ReadFrom(path, 999999)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if len(recs) != 1 {
|
||||
t.Errorf("got %d record(s) — a shrunken file must be re-read from the start", len(recs))
|
||||
}
|
||||
}
|
||||
|
||||
// One unreadable line must not cost the rest of the file, and a record from a
|
||||
// FUTURE format must be skipped rather than guessed at.
|
||||
func TestReadFromSurvivesRubbish(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "peer.ndjson")
|
||||
good, _ := json.Marshal(rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "peer"))
|
||||
future, _ := json.Marshal(Record{V: FormatVersion + 1, Op: OpAdd, UID: "u9", At: time.Now().UTC()})
|
||||
body := append([]byte("this is not json\n"), append(good, '\n')...)
|
||||
body = append(body, append(future, '\n')...)
|
||||
body = append(body, []byte("\n")...) // a blank line
|
||||
must(t, os.WriteFile(path, body, 0o644))
|
||||
|
||||
recs, _, err := ReadFrom(path, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if len(recs) != 1 || recs[0].UID != "u1" {
|
||||
t.Fatalf("got %+v, want only the one readable current-format record", recs)
|
||||
}
|
||||
}
|
||||
|
||||
// Every machine must reach the SAME winner from the same records, whatever
|
||||
// order they arrive in. Ordering on time alone is not even deterministic: two
|
||||
// PCs' clocks are never equal, and a tie would be resolved differently on each
|
||||
// machine, which is how two logs disagree for ever.
|
||||
func TestMergeIsOrderIndependent(t *testing.T) {
|
||||
a := rec("u1", OpUpdate, "2026-08-16T10:00:00Z", 5, "shack")
|
||||
b := rec("u1", OpDelete, "2026-08-16T10:00:00Z", 5, "laptop") // same time AND seq
|
||||
c := rec("u1", OpUpdate, "2026-08-16T09:00:00Z", 9, "shack") // older, higher seq
|
||||
|
||||
forward := Merge([]Record{a, b, c})
|
||||
reverse := Merge([]Record{c, b, a})
|
||||
if forward["u1"].By != reverse["u1"].By || forward["u1"].Op != reverse["u1"].Op {
|
||||
t.Fatalf("order changed the winner: %+v vs %+v", forward["u1"], reverse["u1"])
|
||||
}
|
||||
// Time beats sequence: c is an hour older whatever its counter says.
|
||||
if forward["u1"].At.Equal(c.At) {
|
||||
t.Error("an older change won on its counter — time is the first key")
|
||||
}
|
||||
}
|
||||
|
||||
// A delete is a record like any other, and it must be able to WIN. Without
|
||||
// tombstones a QSO removed on one machine returns on the next sync from
|
||||
// another, for ever.
|
||||
func TestATombstoneCanWin(t *testing.T) {
|
||||
add := rec("u1", OpAdd, "2026-08-16T10:00:00Z", 1, "shack")
|
||||
del := rec("u1", OpDelete, "2026-08-16T11:00:00Z", 1, "laptop")
|
||||
if got := Merge([]Record{add, del})["u1"]; got.Op != OpDelete {
|
||||
t.Errorf("winner is %q — a later deletion must beat an earlier add", got.Op)
|
||||
}
|
||||
// …and an add made AFTER a deletion wins, so re-logging a contact works.
|
||||
readd := rec("u1", OpAdd, "2026-08-16T12:00:00Z", 2, "laptop")
|
||||
if got := Merge([]Record{add, del, readd})["u1"]; got.Op != OpAdd {
|
||||
t.Errorf("winner is %q — a contact logged again after a deletion must come back", got.Op)
|
||||
}
|
||||
}
|
||||
|
||||
// The machine id becomes a FILENAME, on a folder that may be synced between
|
||||
// Windows, Linux and macOS. Anything that cannot be a filename everywhere has
|
||||
// to go.
|
||||
func TestMachineIDIsAUsableFilename(t *testing.T) {
|
||||
for _, name := range []string{"Shack PC", "portable/rig", "Café ☕", "", " ", "a::b*c?", strings.Repeat("x", 60)} {
|
||||
id := NewMachineID(name)
|
||||
for _, bad := range []string{"/", "\\", ":", "*", "?", "\"", "<", ">", "|", " "} {
|
||||
if strings.Contains(id, bad) {
|
||||
t.Errorf("NewMachineID(%q) = %q contains %q", name, id, bad)
|
||||
}
|
||||
}
|
||||
if id == "" || strings.HasPrefix(id, "-") || strings.HasSuffix(id, "-") {
|
||||
t.Errorf("NewMachineID(%q) = %q", name, id)
|
||||
}
|
||||
}
|
||||
// Two machines the operator called the same must not share a file.
|
||||
if NewMachineID("shack") == NewMachineID("shack") {
|
||||
t.Error("two installations named alike produced the same id — they would write to one file")
|
||||
}
|
||||
}
|
||||
|
||||
func must(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user