diff --git a/lib/model/folder.go b/lib/model/folder.go index 7d1f94ff1..ffd167ab1 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -1031,11 +1031,13 @@ func (f *folder) updateLocals(fs []protocol.FileInfo) { } f.forcedRescanPathsMut.Unlock() + seq := f.fset.Sequence(protocol.LocalDeviceID) f.evLogger.Log(events.LocalIndexUpdated, map[string]interface{}{ "folder": f.ID, "items": len(fs), "filenames": filenames, - "version": f.fset.Sequence(protocol.LocalDeviceID), + "sequence": seq, + "version": seq, // legacy for sequence }) } diff --git a/lib/model/model.go b/lib/model/model.go index a7eeb350e..12a7eb621 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -715,15 +715,17 @@ type FolderCompletion struct { GlobalItems int32 NeedItems int32 NeedDeletes int32 + Sequence int64 } -func newFolderCompletion(global, need db.Counts) FolderCompletion { +func newFolderCompletion(global, need db.Counts, sequence int64) FolderCompletion { comp := FolderCompletion{ GlobalBytes: global.Bytes, NeedBytes: need.Bytes, GlobalItems: global.Files + global.Directories + global.Symlinks, NeedItems: need.Files + need.Directories + need.Symlinks, NeedDeletes: need.Deleted, + Sequence: sequence, } comp.setComplectionPct() return comp @@ -764,6 +766,7 @@ func (comp FolderCompletion) Map() map[string]interface{} { "globalItems": comp.GlobalItems, "needItems": comp.NeedItems, "needDeletes": comp.NeedDeletes, + "sequence": comp.Sequence, } } @@ -824,7 +827,7 @@ func (m *model) folderCompletion(device protocol.DeviceID, folder string) Folder need.Bytes = 0 } - comp := newFolderCompletion(global, need) + comp := newFolderCompletion(global, need, snap.Sequence(device)) l.Debugf("%v Completion(%s, %q): %v", m, device, folder, comp.Map()) return comp @@ -974,11 +977,13 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot } files.Update(deviceID, fs) + seq := files.Sequence(deviceID) m.evLogger.Log(events.RemoteIndexUpdated, map[string]interface{}{ - "device": deviceID.String(), - "folder": folder, - "items": len(fs), - "version": files.Sequence(deviceID), + "device": deviceID.String(), + "folder": folder, + "items": len(fs), + "sequence": seq, + "version": seq, // legacy for sequence }) return nil diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 659208cc5..858e5cb9d 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -3929,16 +3929,16 @@ func TestConnectionTerminationOnFolderUnpause(t *testing.T) { func TestAddFolderCompletion(t *testing.T) { // Empty folders are always 100% complete. - comp := newFolderCompletion(db.Counts{}, db.Counts{}) - comp.add(newFolderCompletion(db.Counts{}, db.Counts{})) + comp := newFolderCompletion(db.Counts{}, db.Counts{}, 0) + comp.add(newFolderCompletion(db.Counts{}, db.Counts{}, 0)) if comp.CompletionPct != 100 { t.Error(comp.CompletionPct) } // Completion is of the whole - comp = newFolderCompletion(db.Counts{Bytes: 100}, db.Counts{}) // 100% complete - comp.add(newFolderCompletion(db.Counts{Bytes: 400}, db.Counts{Bytes: 50})) // 82.5% complete - if comp.CompletionPct != 90 { // 100 * (1 - 50/500) + comp = newFolderCompletion(db.Counts{Bytes: 100}, db.Counts{}, 0) // 100% complete + comp.add(newFolderCompletion(db.Counts{Bytes: 400}, db.Counts{Bytes: 50}, 0)) // 82.5% complete + if comp.CompletionPct != 90 { // 100 * (1 - 50/500) t.Error(comp.CompletionPct) } } diff --git a/lib/rc/rc.go b/lib/rc/rc.go index 6825d85a5..f21d0e31d 100644 --- a/lib/rc/rc.go +++ b/lib/rc/rc.go @@ -27,6 +27,7 @@ import ( "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/dialer" "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) @@ -539,19 +540,8 @@ func (p *Process) eventLoop() { case "LocalIndexUpdated": data := ev.Data.(map[string]interface{}) folder := data["folder"].(string) - version, _ := data["version"].(json.Number).Int64() p.eventMut.Lock() - m := p.sequence[folder] - if m == nil { - m = make(map[string]int64) - } - device := p.id.String() - if device == "" { - p.eventMut.Unlock() - panic("race, or startup not complete") - } - m[device] = version - p.sequence[folder] = m + m := p.updateSequenceLocked(folder, p.id.String(), data["sequence"]) p.done[folder] = false l.Debugf("LocalIndexUpdated %v %v done=false\n\t%+v", p.id, folder, m) p.eventMut.Unlock() @@ -560,14 +550,8 @@ func (p *Process) eventLoop() { data := ev.Data.(map[string]interface{}) device := data["device"].(string) folder := data["folder"].(string) - version, _ := data["version"].(json.Number).Int64() p.eventMut.Lock() - m := p.sequence[folder] - if m == nil { - m = make(map[string]int64) - } - m[device] = version - p.sequence[folder] = m + m := p.updateSequenceLocked(folder, device, data["sequence"]) p.done[folder] = false l.Debugf("RemoteIndexUpdated %v %v done=false\n\t%+v", p.id, folder, m) p.eventMut.Unlock() @@ -576,17 +560,38 @@ func (p *Process) eventLoop() { data := ev.Data.(map[string]interface{}) folder := data["folder"].(string) summary := data["summary"].(map[string]interface{}) - need, _ := summary["needBytes"].(json.Number).Int64() + need, _ := summary["needTotalItems"].(json.Number).Int64() done := need == 0 p.eventMut.Lock() + m := p.updateSequenceLocked(folder, p.id.String(), summary["sequence"]) p.done[folder] = done - l.Debugf("Foldersummary %v %v\n\t%+v", p.id, folder, p.done) + l.Debugf("FolderSummary %v %v\n\t%+v\n\t%+v", p.id, folder, p.done, m) + p.eventMut.Unlock() + + case "FolderCompletion": + data := ev.Data.(map[string]interface{}) + device := data["device"].(string) + folder := data["folder"].(string) + p.eventMut.Lock() + m := p.updateSequenceLocked(folder, device, data["sequence"]) + l.Debugf("FolderCompletion %v\n\t%+v", p.id, folder, m) p.eventMut.Unlock() } } } } +func (p *Process) updateSequenceLocked(folder, device string, sequenceIntf interface{}) map[string]int64 { + sequence, _ := sequenceIntf.(json.Number).Int64() + m := p.sequence[folder] + if m == nil { + m = make(map[string]int64) + } + m[device] = sequence + p.sequence[folder] = m + return m +} + type ConnectionStats struct { Address string Type string @@ -658,3 +663,17 @@ func (p *Process) SystemVersion() (SystemVersion, error) { return res, nil } + +func (p *Process) RemoteInSync(folder string, dev protocol.DeviceID) (bool, error) { + bs, err := p.Get(fmt.Sprintf("/rest/db/completion?folder=%v&device=%v", url.QueryEscape(folder), dev)) + if err != nil { + return false, err + } + + var comp model.FolderCompletion + if err := json.Unmarshal(bs, &comp); err != nil { + return false, err + } + + return comp.NeedItems+comp.NeedDeletes == 0, nil +} diff --git a/test/filetype_test.go b/test/filetype_test.go index 8e55accde..caf6fc646 100644 --- a/test/filetype_test.go +++ b/test/filetype_test.go @@ -22,7 +22,7 @@ import ( func TestFileTypeChange(t *testing.T) { // Use no versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{} cfg.SetFolder(fld) @@ -36,7 +36,7 @@ func TestFileTypeChange(t *testing.T) { func TestFileTypeChangeSimpleVersioning(t *testing.T) { // Use simple versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{ Type: "simple", @@ -53,7 +53,7 @@ func TestFileTypeChangeSimpleVersioning(t *testing.T) { func TestFileTypeChangeStaggeredVersioning(t *testing.T) { // Use staggered versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{ Type: "staggered", diff --git a/test/h1/config.xml b/test/h1/config.xml index 94ec05b6e..a658d1bdb 100644 --- a/test/h1/config.xml +++ b/test/h1/config.xml @@ -1,4 +1,4 @@ - + basic @@ -6,7 +6,9 @@ 1 - + + 3600 + 1 0 0 @@ -25,13 +27,18 @@ 0 false standard + standard + false + true basic 1 - + + 3600 + 1 0 0 @@ -50,6 +57,9 @@ 0 false standard + standard + false + true
tcp://127.0.0.1:22004
@@ -144,5 +154,6 @@ default auto 0 + true diff --git a/test/h2/config.xml b/test/h2/config.xml index cefafaea9..6b5552070 100644 --- a/test/h2/config.xml +++ b/test/h2/config.xml @@ -1,11 +1,13 @@ - + basic 1 - + + 3600 + 1 0 0 @@ -24,13 +26,18 @@ 0 false standard + standard + false + true basic 1 - + + 3600 + 1 0 0 @@ -49,13 +56,18 @@ 0 false standard + standard + false + true basic 1 - + + 3600 + 1 0 0 @@ -74,6 +86,9 @@ 0 false standard + standard + false + true
tcp://127.0.0.1:22001
@@ -151,5 +166,6 @@ default auto 0 + true diff --git a/test/h3/config.xml b/test/h3/config.xml index 8c711d94e..ac99d8102 100644 --- a/test/h3/config.xml +++ b/test/h3/config.xml @@ -1,4 +1,4 @@ - + basic @@ -7,6 +7,7 @@ 1 + 3600 1 0 @@ -26,13 +27,18 @@ 0 false standard + standard + false + true basic 1 - + + 3600 + 1 0 0 @@ -51,6 +57,9 @@ 0 false standard + standard + false + true
tcp://127.0.0.1:22001
@@ -128,5 +137,6 @@ default auto 0 + true diff --git a/test/h4/config.xml b/test/h4/config.xml index 9b2a272e5..d0627cd91 100644 --- a/test/h4/config.xml +++ b/test/h4/config.xml @@ -1,9 +1,11 @@ - - + + basic 1 - + + 3600 + 1 2048 0 @@ -17,7 +19,14 @@ false 25 .stfolder - false + false + 0 + 0 + false + standard + standard + false + true
dynamic
@@ -25,12 +34,14 @@ false 0 0 + 0
127.0.0.1:8084
PMA5yUTG-Mw98nJ0YEtWTCHlM5O4aNi0 default + dynamic+https://relays.syncthing.net/endpoint tcp://127.0.0.1:22004 @@ -69,6 +80,14 @@ 0 ~ true - 0 + 0 + https://crash.syncthing.net/newcrash + true + 180 + 20 + default + auto + 0 + true diff --git a/test/override_test.go b/test/override_test.go index c8b0787ea..68b09725f 100644 --- a/test/override_test.go +++ b/test/override_test.go @@ -25,7 +25,7 @@ import ( func TestOverride(t *testing.T) { // Enable "send-only" on s1/default id, _ := protocol.DeviceIDFromString(id1) - cfg, _ := config.Load("h1/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h1/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Type = config.FolderTypeSendOnly cfg.SetFolder(fld) @@ -157,7 +157,7 @@ get to completion when in sendOnly/sendRecv mode. Needs fixing. func TestOverrideIgnores(t *testing.T) { // Enable "sendOnly" on s1/default id, _ := protocol.DeviceIDFromString(id1) - cfg, _ := config.Load("h1/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h1/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.ReadOnly = true cfg.SetFolder(fld) diff --git a/test/reconnect_test.go b/test/reconnect_test.go index 25f499f38..d2557ba99 100644 --- a/test/reconnect_test.go +++ b/test/reconnect_test.go @@ -12,6 +12,8 @@ import ( "log" "testing" "time" + + "github.com/syncthing/syncthing/lib/rc" ) func TestReconnectReceiverDuringTransfer(t *testing.T) { @@ -45,7 +47,7 @@ func testReconnectDuringTransfer(t *testing.T, restartSender, restartReceiver bo receiver := startInstance(t, 2) defer func() { - // We need a receiver over sender, since we'll update it later to + // We need a closure over sender, since we'll update it later to // point at another process. checkedStop(t, receiver) }() @@ -72,7 +74,7 @@ func testReconnectDuringTransfer(t *testing.T, restartSender, restartReceiver bo t.Fatal(err) } - if recv.InSyncBytes > 0 && recv.InSyncBytes == recv.GlobalBytes { + if recv.InSyncBytes > 0 && recv.InSyncBytes == recv.GlobalBytes && rc.InSync("default", receiver, sender) { // Receiver is done break } else if recv.InSyncBytes > prevBytes+recv.GlobalBytes/10 { @@ -115,6 +117,10 @@ func testReconnectDuringTransfer(t *testing.T, restartSender, restartReceiver bo log.Println("Comparing directories...") err = compareDirectories("s1", "s2") if err != nil { - t.Fatal(err) + t.Error(err) + } + + if err := checkRemoteInSync("default", receiver, sender); err != nil { + t.Error(err) } } diff --git a/test/symlink_test.go b/test/symlink_test.go index 13cb2bc70..423c421d6 100644 --- a/test/symlink_test.go +++ b/test/symlink_test.go @@ -26,7 +26,7 @@ func TestSymlinks(t *testing.T) { // Use no versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{} cfg.SetFolder(fld) @@ -44,7 +44,7 @@ func TestSymlinksSimpleVersioning(t *testing.T) { // Use simple versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{ Type: "simple", @@ -65,7 +65,7 @@ func TestSymlinksStaggeredVersioning(t *testing.T) { // Use staggered versioning id, _ := protocol.DeviceIDFromString(id2) - cfg, _ := config.Load("h2/config.xml", id, events.NoopLogger) + cfg, _, _ := config.Load("h2/config.xml", id, events.NoopLogger) fld := cfg.Folders()["default"] fld.Versioning = config.VersioningConfiguration{ Type: "staggered", diff --git a/test/sync_test.go b/test/sync_test.go index 8fe558f36..686ab67c8 100644 --- a/test/sync_test.go +++ b/test/sync_test.go @@ -293,6 +293,19 @@ func scSyncAndCompare(p []*rc.Process, expected [][]fileInfo) error { } } + if err := checkRemoteInSync("default", p[0], p[1]); err != nil { + return err + } + if err := checkRemoteInSync("default", p[0], p[2]); err != nil { + return err + } + if err := checkRemoteInSync(s12Folder, p[0], p[1]); err != nil { + return err + } + if err := checkRemoteInSync("s23", p[1], p[2]); err != nil { + return err + } + return nil } diff --git a/test/util.go b/test/util.go index b1a3aef3b..8865619ac 100644 --- a/test/util.go +++ b/test/util.go @@ -562,3 +562,19 @@ func symlinksSupported() bool { err = os.Symlink("tmp", filepath.Join(tmp, "link")) return err == nil } + +// checkRemoteInSync checks if the devices associated twith the given processes +// are in sync according to the remote status on both sides. +func checkRemoteInSync(folder string, p1, p2 *rc.Process) error { + if inSync, err := p1.RemoteInSync(folder, p2.ID()); err != nil { + return err + } else if !inSync { + return fmt.Errorf(`from device %v folder "%v" is not in sync on device %v`, p1.ID(), folder, p2.ID()) + } + if inSync, err := p2.RemoteInSync(folder, p1.ID()); err != nil { + return err + } else if !inSync { + return fmt.Errorf(`from device %v folder "%v" is not in sync on device %v`, p2.ID(), folder, p1.ID()) + } + return nil +}