Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib: Add mtime window when comparing files (#5852)

This commit is contained in:
Simon Frei 2019-07-23 21:48:53 +02:00 committed by Jakob Borg
parent 35f40e9a58
commit 7a4c88d4e4
12 changed files with 129 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -33,6 +33,7 @@ require (
github.com/prometheus/client_golang v0.9.4
github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9
github.com/sasha-s/go-deadlock v0.2.0
github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/thejerf/suture v3.0.2+incompatible

7
go.sum
View File

@ -4,6 +4,7 @@ github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6 h1:Apvc
github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q=
github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg=
github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@ -37,6 +38,7 @@ github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JY
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d h1:IngNQgbqr5ZOU0exk395Szrvkzes9Ilk1fmJfkw7d+M=
github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
@ -132,6 +134,11 @@ github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 h1:jmLW6izPBVl
github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q=
github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/gopsutil v2.19.6+incompatible h1:49/Gru26Lne9Cl3IoAVDZVM09hvkSrUodgIIsCVRwbs=
github.com/shirou/gopsutil v2.19.6+incompatible/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

30
lib/config/folderconfiguration.go
View File

@ -10,6 +10,10 @@ import (
"errors"
"fmt"
"runtime"
"strings"
"time"
"github.com/shirou/gopsutil/disk"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
@ -53,8 +57,10 @@ type FolderConfiguration struct {
WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
MarkerName string `xml:"markerName" json:"markerName"`
CopyOwnershipFromParent bool `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
RawModTimeWindowS int `xml:"modTimeWindowS" json:"modTimeWindowS"`
cachedFilesystem fs.Filesystem
cachedFilesystem fs.Filesystem
cachedModTimeWindow time.Duration
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@ -111,6 +117,10 @@ func (f FolderConfiguration) Versioner() versioner.Versioner {
return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
}
func (f FolderConfiguration) ModTimeWindow() time.Duration {
return f.cachedModTimeWindow
}
func (f *FolderConfiguration) CreateMarker() error {
if err := f.CheckPath(); err != ErrMarkerMissing {
return err
@ -233,6 +243,24 @@ func (f *FolderConfiguration) prepare() {
if f.MarkerName == "" {
f.MarkerName = DefaultMarkerName
}
switch {
case f.RawModTimeWindowS > 0:
f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
case runtime.GOOS == "android":
usage, err := disk.Usage(f.Filesystem().URI())
if err != nil {
l.Debugf("Error detecting FS at %v on android, setting mtime window to 2s: %v", f.Path, err)
f.cachedModTimeWindow = 2 * time.Second
break
}
if strings.Contains(strings.ToLower(usage.Fstype), "fat") {
l.Debugf("Detecting FS at %v on android, found %v, thus setting mtime window to 2s", f.Path, usage.Fstype)
f.cachedModTimeWindow = 2 * time.Second
break
}
l.Debugf("Detecting FS at %v on android, found %v, thus leaving mtime window at 0", f.Path, usage.Fstype)
}
}
// RequiresRestartOnly returns a copy with only the attributes that require

4
lib/db/db_test.go
View File

@ -167,7 +167,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Unexpected additional file via sequence", f.FileName())
return true
}
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) {
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, 0, true, true, 0) {
found = true
} else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
@ -192,7 +192,7 @@ func TestUpdate0to3(t *testing.T) {
}
f := fi.(protocol.FileInfo)
delete(need, f.Name)
if !f.IsEquivalentOptional(e, true, true, 0) {
if !f.IsEquivalentOptional(e, 0, true, true, 0) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
return true

16
lib/db/set_test.go
View File

@ -905,7 +905,7 @@ func TestWithHaveSequence(t *testing.T) {
i := 2
s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
}
i++
@ -1004,7 +1004,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Error("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(remote0Have[0]) {
} else if !need[0].IsEquivalent(remote0Have[0], 0) {
t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
}
@ -1030,7 +1030,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, remoteDevice0); len(need) != 1 {
t.Error("Expected 1 need for remote 0, got", need)
} else if !need[0].IsEquivalent(localHave[0]) {
} else if !need[0].IsEquivalent(localHave[0], 0) {
t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
}
@ -1066,7 +1066,7 @@ func TestIssue5007(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0]) {
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@ -1101,7 +1101,7 @@ func TestNeedDeleted(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0]) {
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@ -1243,7 +1243,7 @@ func TestNeedAfterUnignore(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need)
} else if !need[0].IsEquivalent(remote) {
} else if !need[0].IsEquivalent(remote, 0) {
t.Fatalf("Got %v, expected %v", need[0], remote)
}
}
@ -1287,7 +1287,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
if len(need) != 1 {
t.Fatal("Locally missing file should be needed")
}
if !need[0].IsEquivalent(file) {
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
@ -1302,7 +1302,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
if len(need) != 1 {
t.Fatal("Locally missing file should be needed regardless of invalid files")
}
if !need[0].IsEquivalent(file) {
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
}

3
lib/model/folder.go
View File

@ -347,6 +347,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
ShortID: f.shortID,
ProgressTickIntervalS: f.ScanProgressIntervalS,
LocalFlags: f.localFlags,
ModTimeWindow: f.ModTimeWindow(),
})
batchFn := func(fs []protocol.FileInfo) error {
@ -365,7 +366,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
switch gf, ok := f.fset.GetGlobal(fs[i].Name); {
case !ok:
continue
case gf.IsEquivalentOptional(fs[i], false, false, protocol.FlagLocalReceiveOnly):
case gf.IsEquivalentOptional(fs[i], f.ModTimeWindow(), false, false, protocol.FlagLocalReceiveOnly):
// What we have locally is equivalent to the global file.
fs[i].Version = fs[i].Version.Merge(gf.Version)
fallthrough

2
lib/model/folder_sendonly.go
View File

@ -74,7 +74,7 @@ func (f *sendOnlyFolder) pull() bool {
}
file := intf.(protocol.FileInfo)
if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) {
if !file.IsEquivalentOptional(curFile, f.ModTimeWindow(), f.IgnorePerms, false, 0) {
return true
}

4
lib/model/folder_sendrecv.go
View File

@ -953,7 +953,7 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
default:
var fi protocol.FileInfo
if fi, err = scanner.CreateFileInfo(stat, target.Name, f.fs); err == nil {
if !fi.IsEquivalentOptional(curTarget, f.IgnorePerms, true, protocol.LocalAllFlags) {
if !fi.IsEquivalentOptional(curTarget, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) {
// Target changed
scanChan <- target.Name
err = errModified
@ -1919,7 +1919,7 @@ func (f *sendReceiveFolder) scanIfItemChanged(stat fs.FileInfo, item protocol.Fi
return errors.Wrap(err, "comparing item on disk to db")
}
if !statItem.IsEquivalentOptional(item, f.IgnorePerms, true, protocol.LocalAllFlags) {
if !statItem.IsEquivalentOptional(item, f.ModTimeWindow(), f.IgnorePerms, true, protocol.LocalAllFlags) {
return errModified
}

52
lib/model/model_test.go
View File

@ -3305,6 +3305,58 @@ func TestConnCloseOnRestart(t *testing.T) {
}
}
func TestModTimeWindow(t *testing.T) {
w, fcfg := tmpDefaultWrapper()
tfs := fcfg.Filesystem()
fcfg.RawModTimeWindowS = 2
w.SetFolder(fcfg)
m := setupModel(w)
defer cleanupModelAndRemoveDir(m, tfs.URI())
name := "foo"
fd, err := tfs.Create(name)
must(t, err)
stat, err := fd.Stat()
must(t, err)
modTime := stat.ModTime()
fd.Close()
m.ScanFolders()
v := protocol.Vector{}
v = v.Update(myID.Short())
fi, ok := m.CurrentFolderFile("default", name)
if !ok {
t.Fatal("File missing")
}
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
err = tfs.Chtimes(name, time.Now(), modTime.Add(time.Second))
must(t, err)
m.ScanFolders()
// No change due to window
fi, _ = m.CurrentFolderFile("default", name)
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
err = tfs.Chtimes(name, time.Now(), modTime.Add(2*time.Second))
must(t, err)
m.ScanFolders()
v = v.Update(myID.Short())
fi, _ = m.CurrentFolderFile("default", name)
if !fi.Version.Equal(v) {
t.Fatalf("Got version %v, expected %v", fi.Version, v)
}
}
func TestDevicePause(t *testing.T) {
sub := events.Default.Subscribe(events.DevicePaused)
defer events.Default.Unsubscribe(sub)

25
lib/protocol/bep_extensions.go
View File

@ -158,12 +158,12 @@ func (f FileInfo) IsEmpty() bool {
return f.Version.Counters == nil
}
func (f FileInfo) IsEquivalent(other FileInfo) bool {
return f.isEquivalent(other, false, false, 0)
func (f FileInfo) IsEquivalent(other FileInfo, modTimeWindow time.Duration) bool {
return f.isEquivalent(other, modTimeWindow, false, false, 0)
}
func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags)
func (f FileInfo) IsEquivalentOptional(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, modTimeWindow, ignorePerms, ignoreBlocks, ignoreFlags)
}
// isEquivalent checks that the two file infos represent the same actual file content,
@ -175,13 +175,13 @@ func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreB
// - invalid flag
// - permissions, unless they are ignored
// A file is not "equivalent", if it has different
// - modification time
// - modification time (difference bigger than modTimeWindow)
// - size
// - blocks, unless there are no blocks to compare (scanning)
// A symlink is not "equivalent", if it has different
// - target
// A directory does not have anything specific to check.
func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
func (f FileInfo) isEquivalent(other FileInfo, modTimeWindow time.Duration, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
if f.MustRescan() || other.MustRescan() {
// These are per definition not equivalent because they don't
// represent a valid state, even if both happen to have the
@ -203,7 +203,7 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
switch f.Type {
case FileInfoTypeFile:
return f.Size == other.Size && f.ModTime().Equal(other.ModTime()) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks))
return f.Size == other.Size && ModTimeEqual(f.ModTime(), other.ModTime(), modTimeWindow) && (ignoreBlocks || BlocksEqual(f.Blocks, other.Blocks))
case FileInfoTypeSymlink:
return f.SymlinkTarget == other.SymlinkTarget
case FileInfoTypeDirectory:
@ -213,6 +213,17 @@ func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
return false
}
func ModTimeEqual(a, b time.Time, modTimeWindow time.Duration) bool {
if a.Equal(b) {
return true
}
diff := a.Sub(b)
if diff < 0 {
diff *= -1
}
return diff < modTimeWindow
}
func PermsEqual(a, b uint32) bool {
switch runtime.GOOS {
case "windows":

4
lib/protocol/protocol_test.go
View File

@ -770,10 +770,10 @@ func TestIsEquivalent(t *testing.T) {
continue
}
if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
if res := tc.a.isEquivalent(tc.b, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
if res := tc.b.isEquivalent(tc.a, 0, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
}

8
lib/scanner/walk.go
View File

@ -54,6 +54,8 @@ type Config struct {
ProgressTickIntervalS int
// Local flags to set on scanned files
LocalFlags uint32
// Modification time is to be considered unchanged if the difference is lower.
ModTimeWindow time.Duration
}
type CurrentFiler interface {
@ -346,7 +348,7 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
f.RawBlockSize = int32(blockSize)
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {
@ -379,7 +381,7 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
f.NoPermissions = w.IgnorePerms
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {
@ -423,7 +425,7 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileIn
f = w.updateFileInfo(f, curFile)
if hasCurFile {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
if curFile.IsEquivalentOptional(f, w.ModTimeWindow, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {