diff --git a/cmd/ursrv/main.go b/cmd/ursrv/main.go index 1c17e6f67..5e6c373f8 100644 --- a/cmd/ursrv/main.go +++ b/cmd/ursrv/main.go @@ -743,6 +743,7 @@ func getReport(db *sql.DB) map[string]interface{} { inc(features["Folder"]["v3"], "Weak hash, always", rep.FolderUsesV3.AlwaysWeakHash) inc(features["Folder"]["v3"], "Weak hash, custom threshold", rep.FolderUsesV3.CustomWeakHashThreshold) inc(features["Folder"]["v3"], "Filesystem watcher", rep.FolderUsesV3.FsWatcherEnabled) + inc(features["Folder"]["v3"], "Case sensitive FS", rep.FolderUsesV3.CaseSensitiveFS) add(featureGroups["Folder"]["v3"], "Conflicts", "Disabled", rep.FolderUsesV3.ConflictsDisabled) add(featureGroups["Folder"]["v3"], "Conflicts", "Unlimited", rep.FolderUsesV3.ConflictsUnlimited) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 6e9d7b788..51c2b29f6 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -22,7 +22,6 @@ import ( "github.com/d4l3k/messagediff" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" ) @@ -129,13 +128,6 @@ func TestDeviceConfig(t *testing.T) { }, } - // The cachedFilesystem will have been resolved to an absolute path, - // depending on where the tests are running. Zero it out so we don't - // fail based on that. - for i := range cfg.Folders { - cfg.Folders[i].cachedFilesystem = nil - } - expectedDevices := []DeviceConfiguration{ { DeviceID: device1, @@ -465,6 +457,7 @@ func TestFolderCheckPath(t *testing.T) { if err != nil { t.Fatal(err) } + testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, n) err = os.MkdirAll(filepath.Join(n, "dir", ".stfolder"), os.FileMode(0777)) if err != nil { @@ -489,7 +482,7 @@ func TestFolderCheckPath(t *testing.T) { }, } - err = osutil.DebugSymlinkForTestsOnly(filepath.Join(n, "dir"), filepath.Join(n, "link")) + err = fs.DebugSymlinkForTestsOnly(testFs, testFs, "dir", "link") if err == nil { t.Log("running with symlink check") testcases = append(testcases, struct { diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index ebd41d86a..3026543a2 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -61,8 +61,8 @@ type FolderConfiguration struct { DisableFsync bool `xml:"disableFsync" json:"disableFsync"` BlockPullOrder BlockPullOrder `xml:"blockPullOrder" json:"blockPullOrder"` CopyRangeMethod fs.CopyRangeMethod `xml:"copyRangeMethod" json:"copyRangeMethod" default:"standard"` + CaseSensitiveFS bool `xml:"caseSensitiveFS" json:"caseSensitiveFS"` - cachedFilesystem fs.Filesystem cachedModTimeWindow time.Duration DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"` @@ -101,11 +101,11 @@ func (f FolderConfiguration) Copy() FolderConfiguration { func (f FolderConfiguration) Filesystem() fs.Filesystem { // This is intentionally not a pointer method, because things like // cfg.Folders["default"].Filesystem() should be valid. - if f.cachedFilesystem == nil { - l.Infoln("bug: uncached filesystem call (should only happen in tests)") - return fs.NewFilesystem(f.FilesystemType, f.Path) + filesystem := fs.NewFilesystem(f.FilesystemType, f.Path) + if !f.CaseSensitiveFS { + filesystem = fs.NewCaseFilesystem(filesystem) } - return f.cachedFilesystem + return filesystem } func (f FolderConfiguration) ModTimeWindow() time.Duration { @@ -210,8 +210,6 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID { } func (f *FolderConfiguration) prepare() { - f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path) - if f.RescanIntervalS > MaxRescanIntervalS { f.RescanIntervalS = MaxRescanIntervalS } else if f.RescanIntervalS < 0 { @@ -263,7 +261,6 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration { // Manual handling for things that are not taken care of by the tag // copier, yet should not cause a restart. - copy.cachedFilesystem = nil blank := FolderConfiguration{} util.CopyMatchingTag(&blank, ©, "restart", func(v string) bool { diff --git a/lib/fs/basicfs_realcaser_unix.go b/lib/fs/basicfs_realcaser_unix.go new file mode 100644 index 000000000..c523cfa39 --- /dev/null +++ b/lib/fs/basicfs_realcaser_unix.go @@ -0,0 +1,13 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// +build !windows + +package fs + +func newBasicRealCaser(fs Filesystem) realCaser { + return newDefaultRealCaser(fs) +} diff --git a/lib/fs/basicfs_realcaser_windows.go b/lib/fs/basicfs_realcaser_windows.go new file mode 100644 index 000000000..d4416f7f7 --- /dev/null +++ b/lib/fs/basicfs_realcaser_windows.go @@ -0,0 +1,58 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// +build windows + +package fs + +import ( + "path/filepath" + "strings" + "syscall" +) + +type basicRealCaserWindows struct { + uri string +} + +func newBasicRealCaser(fs Filesystem) realCaser { + return &basicRealCaserWindows{fs.URI()} +} + +// RealCase returns the correct case for the given name, which is a relative +// path below root, as it exists on disk. +func (r *basicRealCaserWindows) realCase(name string) (string, error) { + if name == "." { + return ".", nil + } + path := r.uri + comps := strings.Split(name, string(PathSeparator)) + var err error + for i, comp := range comps { + path = filepath.Join(path, comp) + comps[i], err = r.realCaseBase(path) + if err != nil { + return "", err + } + } + return filepath.Join(comps...), nil +} + +func (*basicRealCaserWindows) realCaseBase(path string) (string, error) { + p, err := syscall.UTF16PtrFromString(fixLongPath(path)) + if err != nil { + return "", err + } + var fd syscall.Win32finddata + h, err := syscall.FindFirstFile(p, &fd) + if err != nil { + return "", err + } + syscall.FindClose(h) + return syscall.UTF16ToString(fd.FileName[:]), nil +} + +func (r *basicRealCaserWindows) dropCache() {} diff --git a/lib/fs/casefs.go b/lib/fs/casefs.go new file mode 100644 index 000000000..49f127950 --- /dev/null +++ b/lib/fs/casefs.go @@ -0,0 +1,448 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "time" +) + +// Both values were chosen by magic. +const ( + caseCacheTimeout = time.Second + // When the number of names (all lengths of []string from DirNames) + // exceeds this, we drop the cache. + caseMaxCachedNames = 1 << 20 +) + +type ErrCaseConflict struct { + given, real string +} + +func (e *ErrCaseConflict) Error() string { + return fmt.Sprintf(`given name "%v" differs from name in filesystem "%v"`, e.given, e.real) +} + +func IsErrCaseConflict(err error) bool { + e := &ErrCaseConflict{} + return errors.As(err, &e) +} + +type realCaser interface { + realCase(name string) (string, error) + dropCache() +} + +type fskey struct { + fstype FilesystemType + uri string +} + +var ( + caseFilesystems = make(map[fskey]Filesystem) + caseFilesystemsMut sync.Mutex +) + +// caseFilesystem is a BasicFilesystem with additional checks to make a +// potentially case insensitive underlying FS behave like it's case-sensitive. +type caseFilesystem struct { + Filesystem + realCaser +} + +// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem +// behaves like a case-sensitive filesystem. Meaning that it takes into account +// the real casing of a path and returns ErrCaseConflict if the given path differs +// from the real path. It is safe to use with any filesystem, i.e. also a +// case-sensitive one. However it will add some overhead and thus shouldn't be +// used if the filesystem is known to already behave case-sensitively. +func NewCaseFilesystem(fs Filesystem) Filesystem { + caseFilesystemsMut.Lock() + defer caseFilesystemsMut.Unlock() + k := fskey{fs.Type(), fs.URI()} + if caseFs, ok := caseFilesystems[k]; ok { + return caseFs + } + caseFs := &caseFilesystem{ + Filesystem: fs, + } + switch k.fstype { + case FilesystemTypeBasic: + caseFs.realCaser = newBasicRealCaser(fs) + default: + caseFs.realCaser = newDefaultRealCaser(fs) + } + caseFilesystems[k] = caseFs + return caseFs +} + +func (f *caseFilesystem) Chmod(name string, mode FileMode) error { + if err := f.checkCase(name); err != nil { + return err + } + return f.Filesystem.Chmod(name, mode) +} + +func (f *caseFilesystem) Lchown(name string, uid, gid int) error { + if err := f.checkCase(name); err != nil { + return err + } + return f.Filesystem.Lchown(name, uid, gid) +} + +func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error { + if err := f.checkCase(name); err != nil { + return err + } + return f.Filesystem.Chtimes(name, atime, mtime) +} + +func (f *caseFilesystem) Mkdir(name string, perm FileMode) error { + if err := f.checkCase(name); err != nil { + return err + } + if err := f.Filesystem.Mkdir(name, perm); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error { + if err := f.checkCase(path); err != nil { + return err + } + if err := f.Filesystem.MkdirAll(path, perm); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) Lstat(name string) (FileInfo, error) { + var err error + if name, err = Canonicalize(name); err != nil { + return nil, err + } + stat, err := f.Filesystem.Lstat(name) + if err != nil { + return nil, err + } + if err = f.checkCaseExisting(name); err != nil { + return nil, err + } + return stat, nil +} + +func (f *caseFilesystem) Remove(name string) error { + if err := f.checkCase(name); err != nil { + return err + } + if err := f.Filesystem.Remove(name); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) RemoveAll(name string) error { + if err := f.checkCase(name); err != nil { + return err + } + if err := f.Filesystem.RemoveAll(name); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) Rename(oldpath, newpath string) error { + if err := f.checkCase(oldpath); err != nil { + return err + } + if err := f.Filesystem.Rename(oldpath, newpath); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) Stat(name string) (FileInfo, error) { + var err error + if name, err = Canonicalize(name); err != nil { + return nil, err + } + stat, err := f.Filesystem.Stat(name) + if err != nil { + return nil, err + } + if err = f.checkCaseExisting(name); err != nil { + return nil, err + } + return stat, nil +} + +func (f *caseFilesystem) DirNames(name string) ([]string, error) { + if err := f.checkCase(name); err != nil { + return nil, err + } + return f.Filesystem.DirNames(name) +} + +func (f *caseFilesystem) Open(name string) (File, error) { + if err := f.checkCase(name); err != nil { + return nil, err + } + return f.Filesystem.Open(name) +} + +func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) { + if err := f.checkCase(name); err != nil { + return nil, err + } + file, err := f.Filesystem.OpenFile(name, flags, mode) + if err != nil { + return nil, err + } + f.dropCache() + return file, nil +} + +func (f *caseFilesystem) ReadSymlink(name string) (string, error) { + if err := f.checkCase(name); err != nil { + return "", err + } + return f.Filesystem.ReadSymlink(name) +} + +func (f *caseFilesystem) Create(name string) (File, error) { + if err := f.checkCase(name); err != nil { + return nil, err + } + file, err := f.Filesystem.Create(name) + if err != nil { + return nil, err + } + f.dropCache() + return file, nil +} + +func (f *caseFilesystem) CreateSymlink(target, name string) error { + if err := f.checkCase(name); err != nil { + return err + } + if err := f.Filesystem.CreateSymlink(target, name); err != nil { + return err + } + f.dropCache() + return nil +} + +func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error { + // Walking the filesystem is likely (in Syncthing's case certainly) done + // to pick up external changes, for which caching is undesirable. + f.dropCache() + if err := f.checkCase(root); err != nil { + return err + } + return f.Filesystem.Walk(root, walkFn) +} + +func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) { + if err := f.checkCase(path); err != nil { + return nil, nil, err + } + return f.Filesystem.Watch(path, ignore, ctx, ignorePerms) +} + +func (f *caseFilesystem) Hide(name string) error { + if err := f.checkCase(name); err != nil { + return err + } + return f.Filesystem.Hide(name) +} + +func (f *caseFilesystem) Unhide(name string) error { + if err := f.checkCase(name); err != nil { + return err + } + return f.Filesystem.Unhide(name) +} + +func (f *caseFilesystem) checkCase(name string) error { + var err error + if name, err = Canonicalize(name); err != nil { + return err + } + // Stat is necessary for case sensitive FS, as it's then not a conflict + // if name is e.g. "foo" and on dir there is "Foo". + if _, err := f.Filesystem.Lstat(name); err != nil { + if IsNotExist(err) { + return nil + } + return err + } + return f.checkCaseExisting(name) +} + +// checkCaseExisting must only be called after successfully canonicalizing and +// stating the file. +func (f *caseFilesystem) checkCaseExisting(name string) error { + realName, err := f.realCase(name) + if IsNotExist(err) { + // It did exist just before -> cache is outdated, try again + f.dropCache() + realName, err = f.realCase(name) + } + if err != nil { + return err + } + if realName != name { + return &ErrCaseConflict{name, realName} + } + return nil +} + +type defaultRealCaser struct { + fs Filesystem + root *caseNode + count int + timer *time.Timer + timerStop chan struct{} + mut sync.RWMutex +} + +func newDefaultRealCaser(fs Filesystem) *defaultRealCaser { + caser := &defaultRealCaser{ + fs: fs, + root: &caseNode{name: "."}, + timer: time.NewTimer(0), + } + <-caser.timer.C + return caser +} + +func (r *defaultRealCaser) realCase(name string) (string, error) { + out := "." + if name == out { + return out, nil + } + + r.mut.Lock() + defer func() { + if r.count > caseMaxCachedNames { + select { + case r.timerStop <- struct{}{}: + default: + } + r.dropCacheLocked() + } + r.mut.Unlock() + }() + + node := r.root + for _, comp := range strings.Split(name, string(PathSeparator)) { + if node.dirNames == nil { + // Haven't called DirNames yet + var err error + node.dirNames, err = r.fs.DirNames(out) + if err != nil { + return "", err + } + node.dirNamesLower = make([]string, len(node.dirNames)) + for i, n := range node.dirNames { + node.dirNamesLower[i] = UnicodeLowercase(n) + } + node.children = make(map[string]*caseNode) + node.results = make(map[string]*caseNode) + r.count += len(node.dirNames) + } else if child, ok := node.results[comp]; ok { + // Check if this exact name has been queried before to shortcut + node = child + out = filepath.Join(out, child.name) + continue + } + // Actually loop dirNames to search for a match + n, err := findCaseInsensitiveMatch(comp, node.dirNames, node.dirNamesLower) + if err != nil { + return "", err + } + child, ok := node.children[n] + if !ok { + child = &caseNode{name: n} + } + node.results[comp] = child + node.children[n] = child + node = child + out = filepath.Join(out, n) + } + + return out, nil +} + +func (r *defaultRealCaser) startCaseResetTimerLocked() { + r.timerStop = make(chan struct{}) + r.timer.Reset(caseCacheTimeout) + go func() { + select { + case <-r.timer.C: + r.dropCache() + case <-r.timerStop: + if !r.timer.Stop() { + <-r.timer.C + } + r.mut.Lock() + r.timerStop = nil + r.mut.Unlock() + } + }() +} + +func (r *defaultRealCaser) dropCache() { + r.mut.Lock() + r.dropCacheLocked() + r.mut.Unlock() +} + +func (r *defaultRealCaser) dropCacheLocked() { + r.root = &caseNode{name: "."} + r.count = 0 +} + +// Both name and the key to children are "Real", case resolved names of the path +// component this node represents (i.e. containing no path separator). +// The key to results is also a path component, but as given to RealCase, not +// case resolved. +type caseNode struct { + name string + dirNames []string + dirNamesLower []string + children map[string]*caseNode + results map[string]*caseNode +} + +func findCaseInsensitiveMatch(name string, names, namesLower []string) (string, error) { + lower := UnicodeLowercase(name) + candidate := "" + for i, n := range names { + if n == name { + return n, nil + } + if candidate == "" && namesLower[i] == lower { + candidate = n + } + } + if candidate == "" { + return "", ErrNotExist + } + return candidate, nil +} diff --git a/lib/fs/casefs_test.go b/lib/fs/casefs_test.go new file mode 100644 index 000000000..a708c7d59 --- /dev/null +++ b/lib/fs/casefs_test.go @@ -0,0 +1,279 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package fs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestRealCase(t *testing.T) { + // Verify realCase lookups on various underlying filesystems. + + t.Run("fake-sensitive", func(t *testing.T) { + testRealCase(t, newFakeFilesystem(t.Name())) + }) + t.Run("fake-insensitive", func(t *testing.T) { + testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true")) + }) + t.Run("actual", func(t *testing.T) { + fsys, tmpDir := setup(t) + defer os.RemoveAll(tmpDir) + testRealCase(t, fsys) + }) +} + +func testRealCase(t *testing.T, fsys Filesystem) { + testFs := NewCaseFilesystem(fsys).(*caseFilesystem) + comps := []string{"Foo", "bar", "BAZ", "bAs"} + path := filepath.Join(comps...) + testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777) + fd, err := testFs.Create(path) + if err != nil { + t.Fatal(err) + } + fd.Close() + + for i, tc := range []struct { + in string + len int + }{ + {path, 4}, + {strings.ToLower(path), 4}, + {strings.ToUpper(path), 4}, + {"foo", 1}, + {"FOO", 1}, + {"foO", 1}, + {filepath.Join("Foo", "bar"), 2}, + {filepath.Join("Foo", "bAr"), 2}, + {filepath.Join("FoO", "bar"), 2}, + {filepath.Join("foo", "bar", "BAZ"), 3}, + {filepath.Join("Foo", "bar", "bAz"), 3}, + {filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose + } { + out, err := testFs.realCase(tc.in) + if err != nil { + t.Error(err) + } else if exp := filepath.Join(comps[:tc.len]...); out != exp { + t.Errorf("tc %v: Expected %v, got %v", i, exp, out) + } + } +} + +func TestRealCaseSensitive(t *testing.T) { + // Verify that realCase returns the best on-disk case for case sensitive + // systems. Test is skipped if the underlying fs is insensitive. + + t.Run("fake-sensitive", func(t *testing.T) { + testRealCaseSensitive(t, newFakeFilesystem(t.Name())) + }) + t.Run("actual", func(t *testing.T) { + fsys, tmpDir := setup(t) + defer os.RemoveAll(tmpDir) + testRealCaseSensitive(t, fsys) + }) +} + +func testRealCaseSensitive(t *testing.T, fsys Filesystem) { + testFs := NewCaseFilesystem(fsys).(*caseFilesystem) + + names := make([]string, 2) + names[0] = "foo" + names[1] = strings.ToUpper(names[0]) + for _, n := range names { + if err := testFs.MkdirAll(n, 0777); err != nil { + if IsErrCaseConflict(err) { + t.Skip("Filesystem is case-insensitive") + } + t.Fatal(err) + } + } + + for _, n := range names { + if rn, err := testFs.realCase(n); err != nil { + t.Error(err) + } else if rn != n { + t.Errorf("Got %v, expected %v", rn, n) + } + } +} + +func TestCaseFSStat(t *testing.T) { + // Verify that a Stat() lookup behaves in a case sensitive manner + // regardless of the underlying fs. + + t.Run("fake-sensitive", func(t *testing.T) { + testCaseFSStat(t, newFakeFilesystem(t.Name())) + }) + t.Run("fake-insensitive", func(t *testing.T) { + testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true")) + }) + t.Run("actual", func(t *testing.T) { + fsys, tmpDir := setup(t) + defer os.RemoveAll(tmpDir) + testCaseFSStat(t, fsys) + }) +} + +func testCaseFSStat(t *testing.T, fsys Filesystem) { + fd, err := fsys.Create("foo") + if err != nil { + t.Fatal(err) + } + fd.Close() + + // Check if the underlying fs is sensitive or not + sensitive := true + if _, err = fsys.Stat("FOO"); err == nil { + sensitive = false + } + + testFs := NewCaseFilesystem(fsys) + _, err = testFs.Stat("FOO") + if sensitive { + if IsNotExist(err) { + t.Log("pass: case sensitive underlying fs") + } else { + t.Error("expected NotExist, not", err, "for sensitive fs") + } + } else if IsErrCaseConflict(err) { + t.Log("pass: case insensitive underlying fs") + } else { + t.Error("expected ErrCaseConflict, not", err, "for insensitive fs") + } +} + +func BenchmarkWalkCaseFakeFS10k(b *testing.B) { + fsys, paths, err := fakefsForBenchmark(10_000, 0) + if err != nil { + b.Fatal(err) + } + slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond) + if err != nil { + b.Fatal(err) + } + b.Run("raw-fastfs", func(b *testing.B) { + benchmarkWalkFakeFS(b, fsys, paths) + b.ReportAllocs() + }) + b.Run("case-fastfs", func(b *testing.B) { + benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths) + b.ReportAllocs() + }) + b.Run("raw-slowfs", func(b *testing.B) { + benchmarkWalkFakeFS(b, slowsys, paths) + b.ReportAllocs() + }) + b.Run("case-slowfs", func(b *testing.B) { + benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths) + b.ReportAllocs() + }) +} + +func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) { + // Simulate a scanner pass over the filesystem. First walk it to + // discover all names, then stat each name individually to check if it's + // been deleted or not (pretending that they all existed in the + // database). + + var ms0 runtime.MemStats + runtime.ReadMemStats(&ms0) + t0 := time.Now() + + for i := 0; i < b.N; i++ { + if err := doubleWalkFS(fsys, paths); err != nil { + b.Fatal(err) + } + } + + t1 := time.Now() + var ms1 runtime.MemStats + runtime.ReadMemStats(&ms1) + + // We add metrics per path entry + b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry") + b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry") + b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry") +} + +func TestStressCaseFS(t *testing.T) { + // Exercise a bunch of paralell operations for stressing out race + // conditions in the realnamer cache etc. + + const limit = 10 * time.Second + if testing.Short() { + t.Skip("long test") + } + + fsys, paths, err := fakefsForBenchmark(10_000, 0) + if err != nil { + t.Fatal(err) + } + for i := 0; i < runtime.NumCPU()/2+1; i++ { + t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) { + // Walk the filesystem and stat everything + t.Parallel() + t0 := time.Now() + for time.Since(t0) < limit { + if err := doubleWalkFS(fsys, paths); err != nil { + t.Fatal(err) + } + } + }) + t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) { + // Touch all the things + t.Parallel() + t0 := time.Now() + for time.Since(t0) < limit { + for _, p := range paths { + now := time.Now() + if err := fsys.Chtimes(p, now, now); err != nil { + t.Fatal(err) + } + } + } + }) + } +} + +func doubleWalkFS(fsys Filesystem, paths []string) error { + if err := fsys.Walk("/", func(path string, info FileInfo, err error) error { + return err + }); err != nil { + return err + } + + for _, p := range paths { + if _, err := fsys.Lstat(p); err != nil { + return err + } + } + return nil +} + +func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) { + fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency)) + + var paths []string + if err := fsys.Walk("/", func(path string, info FileInfo, err error) error { + paths = append(paths, path) + return err + }); err != nil { + return nil, nil, err + } + if len(paths) < nfiles { + return nil, nil, errors.New("didn't find enough stuff") + } + + return fsys, paths, nil +} diff --git a/lib/fs/debug_symlink_unix.go b/lib/fs/debug_symlink_unix.go new file mode 100644 index 000000000..d09f38c44 --- /dev/null +++ b/lib/fs/debug_symlink_unix.go @@ -0,0 +1,47 @@ +// Copyright (C) 2017 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// +build !windows + +package fs + +import ( + "os" + "path/filepath" +) + +// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code, +// hence the cumbersome name to make it obvious if this ever leaks. Its +// reason for existence is the Windows version, which allows creating +// symlinks when non-elevated. +func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error { + if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok { + if err := caseFs.checkCase(newname); err != nil { + return err + } + caseFs.dropCache() + } + if err := os.Symlink(filepath.Join(oldFs.URI(), oldname), filepath.Join(newFs.URI(), newname)); err != nil { + return err + } + return nil +} + +// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem. +func unwrapFilesystem(fs Filesystem) Filesystem { + for { + switch sfs := fs.(type) { + case *logFilesystem: + fs = sfs.Filesystem + case *walkFilesystem: + fs = sfs.Filesystem + case *MtimeFS: + fs = sfs.Filesystem + default: + return sfs + } + } +} diff --git a/lib/osutil/symlink_windows.go b/lib/fs/debug_symlink_windows.go similarity index 95% rename from lib/osutil/symlink_windows.go rename to lib/fs/debug_symlink_windows.go index aaf4b86ac..c21989d30 100644 --- a/lib/osutil/symlink_windows.go +++ b/lib/fs/debug_symlink_windows.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package osutil +package fs import ( "os" @@ -17,7 +17,10 @@ import ( // This is not and should not be used in Syncthing code, hence the // cumbersome name to make it obvious if this ever leaks. Nonetheless it's // useful in tests. -func DebugSymlinkForTestsOnly(oldname, newname string) error { +func DebugSymlinkForTestsOnly(oldFs, newFS Filesystem, oldname, newname string) error { + oldname = filepath.Join(oldFs.URI(), oldname) + newname = filepath.Join(newFS.URI(), newname) + // CreateSymbolicLink is not supported before Windows Vista if syscall.LoadCreateSymbolicLink() != nil { return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS} diff --git a/lib/fs/fakefs.go b/lib/fs/fakefs.go index df82edaae..7f29ab327 100644 --- a/lib/fs/fakefs.go +++ b/lib/fs/fakefs.go @@ -48,14 +48,17 @@ const randomBlockShift = 14 // 128k // sizeavg=n to set the average size of random files, in bytes (default 1<<20) // seed=n to set the initial random seed (default 0) // insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false) +// latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format // // - Two fakefs:s pointing at the same root path see the same files. // type fakefs struct { + uri string mut sync.Mutex root *fakeEntry insens bool withContent bool + latency time.Duration } var ( @@ -63,23 +66,25 @@ var ( fakefsFs = make(map[string]*fakefs) ) -func newFakeFilesystem(root string) *fakefs { +func newFakeFilesystem(rootURI string) *fakefs { fakefsMut.Lock() defer fakefsMut.Unlock() + root := rootURI var params url.Values - uri, err := url.Parse(root) + uri, err := url.Parse(rootURI) if err == nil { root = uri.Path params = uri.Query() } - if fs, ok := fakefsFs[root]; ok { + if fs, ok := fakefsFs[rootURI]; ok { // Already have an fs at this path return fs } fs := &fakefs{ + uri: "fake://" + rootURI, root: &fakeEntry{ name: "/", entryType: fakeEntryTypeDir, @@ -129,6 +134,10 @@ func newFakeFilesystem(root string) *fakefs { // Also create a default folder marker for good measure fs.Mkdir(".stfolder", 0700) + // We only set the latency after doing the operations required to create + // the filesystem initially. + fs.latency, _ = time.ParseDuration(params.Get("latency")) + fakefsFs[root] = fs return fs } @@ -185,6 +194,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry { func (fs *fakefs) Chmod(name string, mode FileMode) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { return os.ErrNotExist @@ -196,6 +206,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error { func (fs *fakefs) Lchown(name string, uid, gid int) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { return os.ErrNotExist @@ -208,6 +219,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error { func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { return os.ErrNotExist @@ -219,6 +231,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error { func (fs *fakefs) create(name string) (*fakeEntry, error) { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) if entry := fs.entryForName(name); entry != nil { if entry.entryType == fakeEntryTypeDir { @@ -284,6 +297,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error { func (fs *fakefs) DirNames(name string) ([]string, error) { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { @@ -301,6 +315,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) { func (fs *fakefs) Lstat(name string) (FileInfo, error) { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { @@ -318,6 +333,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) { func (fs *fakefs) Mkdir(name string, perm FileMode) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) dir := filepath.Dir(name) base := filepath.Base(name) @@ -348,6 +364,10 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error { } func (fs *fakefs) MkdirAll(name string, perm FileMode) error { + fs.mut.Lock() + defer fs.mut.Unlock() + time.Sleep(fs.latency) + name = filepath.ToSlash(name) name = strings.Trim(name, "/") comps := strings.Split(name, "/") @@ -382,6 +402,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error { func (fs *fakefs) Open(name string) (File, error) { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil || entry.entryType != fakeEntryTypeFile { @@ -401,6 +422,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error) fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) dir := filepath.Dir(name) base := filepath.Base(name) @@ -438,6 +460,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error) func (fs *fakefs) ReadSymlink(name string) (string, error) { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) entry := fs.entryForName(name) if entry == nil { @@ -451,6 +474,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) { func (fs *fakefs) Remove(name string) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) if fs.insens { name = UnicodeLowercase(name) @@ -472,6 +496,7 @@ func (fs *fakefs) Remove(name string) error { func (fs *fakefs) RemoveAll(name string) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) if fs.insens { name = UnicodeLowercase(name) @@ -491,6 +516,7 @@ func (fs *fakefs) RemoveAll(name string) error { func (fs *fakefs) Rename(oldname, newname string) error { fs.mut.Lock() defer fs.mut.Unlock() + time.Sleep(fs.latency) oldKey := filepath.Base(oldname) newKey := filepath.Base(newname) @@ -578,7 +604,7 @@ func (fs *fakefs) Type() FilesystemType { } func (fs *fakefs) URI() string { - return "fake://" + fs.root.name + return fs.uri } func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool { diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 8fe3ece20..5e32375ee 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -129,9 +129,9 @@ type sendReceiveFolder struct { blockPullReorderer blockPullReorderer writeLimiter *byteSemaphore - pullErrors map[string]string // errors for most recent/current iteration - oldPullErrors map[string]string // errors from previous iterations for log filtering only - pullErrorsMut sync.Mutex + pullErrors map[string]string // actual exposed pull errors + tempPullErrors map[string]string // pull errors that might be just transient + pullErrorsMut sync.Mutex } func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service { @@ -192,6 +192,10 @@ func (f *sendReceiveFolder) pull() bool { changed := 0 + f.pullErrorsMut.Lock() + f.pullErrors = nil + f.pullErrorsMut.Unlock() + for tries := 0; tries < maxPullerIterations; tries++ { select { case <-f.ctx.Done(): @@ -216,8 +220,14 @@ func (f *sendReceiveFolder) pull() bool { } f.pullErrorsMut.Lock() + f.pullErrors = f.tempPullErrors + f.tempPullErrors = nil + for path, err := range f.pullErrors { + l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err) + } pullErrNum := len(f.pullErrors) f.pullErrorsMut.Unlock() + if pullErrNum > 0 { l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum) f.evLogger.Log(events.FolderErrors, map[string]interface{}{ @@ -235,8 +245,7 @@ func (f *sendReceiveFolder) pull() bool { // flagged as needed in the folder. func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { f.pullErrorsMut.Lock() - f.oldPullErrors = f.pullErrors - f.pullErrors = make(map[string]string) + f.tempPullErrors = make(map[string]string) f.pullErrorsMut.Unlock() snap := f.fset.Snapshot() @@ -306,10 +315,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int { close(dbUpdateChan) updateWg.Wait() - f.pullErrorsMut.Lock() - f.oldPullErrors = nil - f.pullErrorsMut.Unlock() - f.queue.Reset() return changed @@ -739,7 +744,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps } // There is already something under that name, we need to handle that. - if info, err := f.fs.Lstat(file.Name); err == nil { + switch info, err := f.fs.Lstat(file.Name); { + case err != nil && !fs.IsNotExist(err): + f.newPullError(file.Name, errors.Wrap(err, "checking for existing symlink")) + return + case err == nil: // Check that it is what we have in the database. curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name) if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil { @@ -1783,7 +1792,7 @@ func (f *sendReceiveFolder) newPullError(path string, err error) { // We might get more than one error report for a file (i.e. error on // Write() followed by Close()); we keep the first error as that is // probably closer to the root cause. - if _, ok := f.pullErrors[path]; ok { + if _, ok := f.tempPullErrors[path]; ok { return } @@ -1791,15 +1800,9 @@ func (f *sendReceiveFolder) newPullError(path string, err error) { // Use "syncing" as opposed to "pulling" as the latter might be used // for errors occurring specificly in the puller routine. errStr := fmt.Sprintln("syncing:", err) - f.pullErrors[path] = errStr + f.tempPullErrors[path] = errStr - if oldErr, ok := f.oldPullErrors[path]; ok && oldErr == errStr { - l.Debugf("Repeat error on puller (folder %s, item %q): %v", f.Description(), path, err) - delete(f.oldPullErrors, path) // Potential repeats are now caught by f.pullErrors itself - return - } - - l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err) + l.Debugf("%v new error for %v: %v", f, path, err) } func (f *sendReceiveFolder) Errors() []FileError { diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index b42fe7dc8..2615521e3 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -10,11 +10,13 @@ import ( "bytes" "context" "crypto/rand" + "errors" "io" "io/ioutil" "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -95,6 +97,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol model.Supervisor.Stop() f := model.folderRunners[fcfg.ID].(*sendReceiveFolder) f.pullErrors = make(map[string]string) + f.tempPullErrors = make(map[string]string) f.ctx = context.Background() // Update index @@ -983,7 +986,7 @@ func TestDeleteBehindSymlink(t *testing.T) { must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file")) must(t, ffs.RemoveAll(link)) - if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil { + if err := fs.DebugSymlinkForTestsOnly(destFs, ffs, "", link); err != nil { if runtime.GOOS == "windows" { // Probably we require permissions we don't have. t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error()) @@ -1087,6 +1090,122 @@ func TestPullDeleteUnscannedDir(t *testing.T) { } } +func TestPullCaseOnlyPerformFinish(t *testing.T) { + m, f := setupSendReceiveFolder() + defer cleanupSRFolder(f, m) + ffs := f.Filesystem() + + name := "foo" + contents := []byte("contents") + must(t, writeFile(ffs, name, contents, 0644)) + must(t, f.scanSubdirs(nil)) + + var cur protocol.FileInfo + hasCur := false + snap := dbSnapshot(t, m, f.ID) + defer snap.Release() + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { + if hasCur { + t.Fatal("got more than one file") + } + cur = i.(protocol.FileInfo) + hasCur = true + return true + }) + if !hasCur { + t.Fatal("file is missing") + } + + remote := *(&cur) + remote.Version = protocol.Vector{}.Update(device1.Short()) + remote.Name = strings.ToUpper(cur.Name) + temp := fs.TempName(remote.Name) + must(t, writeFile(ffs, temp, contents, 0644)) + scanChan := make(chan string, 1) + dbUpdateChan := make(chan dbUpdateJob, 1) + + err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan) + + select { + case <-dbUpdateChan: // boring case sensitive filesystem + return + case <-scanChan: + t.Error("no need to scan anything here") + default: + } + + var caseErr *fs.ErrCaseConflict + if !errors.As(err, &caseErr) { + t.Error("Expected case conflict error, got", err) + } +} + +func TestPullCaseOnlyDir(t *testing.T) { + testPullCaseOnlyDirOrSymlink(t, true) +} + +func TestPullCaseOnlySymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks not supported on windows") + } + testPullCaseOnlyDirOrSymlink(t, false) +} + +func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) { + m, f := setupSendReceiveFolder() + defer cleanupSRFolder(f, m) + ffs := f.Filesystem() + + name := "foo" + if dir { + must(t, ffs.Mkdir(name, 0777)) + } else { + must(t, ffs.CreateSymlink("target", name)) + } + + must(t, f.scanSubdirs(nil)) + var cur protocol.FileInfo + hasCur := false + snap := dbSnapshot(t, m, f.ID) + defer snap.Release() + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { + if hasCur { + t.Fatal("got more than one file") + } + cur = i.(protocol.FileInfo) + hasCur = true + return true + }) + if !hasCur { + t.Fatal("file is missing") + } + + scanChan := make(chan string, 1) + dbUpdateChan := make(chan dbUpdateJob, 1) + remote := *(&cur) + remote.Version = protocol.Vector{}.Update(device1.Short()) + remote.Name = strings.ToUpper(cur.Name) + + if dir { + f.handleDir(remote, snap, dbUpdateChan, scanChan) + } else { + f.handleSymlink(remote, snap, dbUpdateChan, scanChan) + } + + select { + case <-dbUpdateChan: // boring case sensitive filesystem + return + case <-scanChan: + t.Error("no need to scan anything here") + default: + } + if errStr, ok := f.tempPullErrors[remote.Name]; !ok { + t.Error("missing error for", remote.Name) + } else if !strings.Contains(errStr, "differs from name") { + t.Error("unexpected error", errStr, "for", remote.Name) + } +} + func cleanupSharedPullerState(s *sharedPullerState) { s.mut.Lock() defer s.mut.Unlock() diff --git a/lib/model/folder_test.go b/lib/model/folder_test.go index a13fd9eb6..237cb2aaf 100644 --- a/lib/model/folder_test.go +++ b/lib/model/folder_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/d4l3k/messagediff" + "github.com/syncthing/syncthing/lib/config" ) diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 137ad6dfc..242db0a07 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -270,17 +270,15 @@ func BenchmarkRequestOut(b *testing.B) { } func BenchmarkRequestInSingleFile(b *testing.B) { - testOs := &fatalOs{b} - m := setupModel(defaultCfgWrapper) defer cleanupModel(m) buf := make([]byte, 128<<10) rand.Read(buf) - testOs.RemoveAll("testdata/request") - defer testOs.RemoveAll("testdata/request") - testOs.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755) - ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644) + mustRemove(b, defaultFs.RemoveAll("request")) + defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }() + must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755)) + writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644) b.ResetTimer() @@ -294,13 +292,11 @@ func BenchmarkRequestInSingleFile(b *testing.B) { } func TestDeviceRename(t *testing.T) { - testOs := &fatalOs{t} - hello := protocol.HelloResult{ ClientName: "syncthing", ClientVersion: "v0.9.4", } - defer testOs.Remove("testdata/tmpconfig.xml") + defer func() { mustRemove(t, defaultFs.Remove("tmpconfig.xml")) }() rawCfg := config.New(device1) rawCfg.Devices = []config.DeviceConfiguration{ @@ -1447,12 +1443,10 @@ func changeIgnores(t *testing.T, m *model, expected []string) { } func TestIgnores(t *testing.T) { - testOs := &fatalOs{t} - // Assure a clean start state - testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName)) - testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644) - ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644) + mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) + mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) + writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644) m := setupModel(defaultCfgWrapper) defer cleanupModel(m) @@ -1504,18 +1498,16 @@ func TestIgnores(t *testing.T) { // Make sure no .stignore file is considered valid defer func() { - testOs.Rename("testdata/.stignore.bak", "testdata/.stignore") + must(t, defaultFs.Rename(".stignore.bak", ".stignore")) }() - testOs.Rename("testdata/.stignore", "testdata/.stignore.bak") + must(t, defaultFs.Rename(".stignore", ".stignore.bak")) changeIgnores(t, m, []string{}) } func TestEmptyIgnores(t *testing.T) { - testOs := &fatalOs{t} - // Assure a clean start state - testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName)) - testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644) + mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) + must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) m := setupModel(defaultCfgWrapper) defer cleanupModel(m) @@ -2117,14 +2109,14 @@ func benchmarkTree(b *testing.B, n1, n2 int) { } func TestIssue3028(t *testing.T) { - testOs := &fatalOs{t} - // Create two files that we'll delete, one with a name that is a prefix of the other. - must(t, ioutil.WriteFile("testdata/testrm", []byte("Hello"), 0644)) - defer testOs.Remove("testdata/testrm") - must(t, ioutil.WriteFile("testdata/testrm2", []byte("Hello"), 0644)) - defer testOs.Remove("testdata/testrm2") + must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644)) + must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644)) + defer func() { + mustRemove(t, defaultFs.Remove("testrm")) + mustRemove(t, defaultFs.Remove("testrm2")) + }() // Create a model and default folder @@ -2138,8 +2130,8 @@ func TestIssue3028(t *testing.T) { // Delete and rescan specifically these two - testOs.Remove("testdata/testrm") - testOs.Remove("testdata/testrm2") + must(t, defaultFs.Remove("testrm")) + must(t, defaultFs.Remove("testrm2")) m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"}) // Verify that the number of files decreased by two and the number of @@ -2601,7 +2593,7 @@ func TestIssue2571(t *testing.T) { must(t, testFs.RemoveAll("toLink")) - must(t, osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "linkTarget"), filepath.Join(testFs.URI(), "toLink"))) + must(t, fs.DebugSymlinkForTestsOnly(testFs, testFs, "linkTarget", "toLink")) m.ScanFolder("default") @@ -2718,13 +2710,10 @@ func TestCustomMarkerName(t *testing.T) { {Name: "dummyfile"}, }) - fcfg := config.FolderConfiguration{ - ID: "default", - Path: "rwtestfolder", - Type: config.FolderTypeSendReceive, - RescanIntervalS: 1, - MarkerName: "myfile", - } + fcfg := testFolderConfigTmp() + fcfg.ID = "default" + fcfg.RescanIntervalS = 1 + fcfg.MarkerName = "myfile" cfg := createTmpWrapper(config.Configuration{ Folders: []config.FolderConfiguration{fcfg}, Devices: []config.DeviceConfiguration{ @@ -2735,13 +2724,12 @@ func TestCustomMarkerName(t *testing.T) { }) testOs.RemoveAll(fcfg.Path) - defer testOs.RemoveAll(fcfg.Path) m := newModel(cfg, myID, "syncthing", "dev", ldb, nil) sub := m.evLogger.Subscribe(events.StateChanged) defer sub.Unsubscribe() m.ServeBackground() - defer cleanupModel(m) + defer cleanupModelAndRemoveDir(m, fcfg.Path) waitForState(t, sub, "default", "folder path missing") @@ -3806,6 +3794,57 @@ func TestBlockListMap(t *testing.T) { } } +func TestScanRenameCaseOnly(t *testing.T) { + wcfg, fcfg := tmpDefaultWrapper() + m := setupModel(wcfg) + defer cleanupModel(m) + + ffs := fcfg.Filesystem() + name := "foo" + must(t, writeFile(ffs, name, []byte("contents"), 0644)) + + m.ScanFolders() + + snap := dbSnapshot(t, m, fcfg.ID) + defer snap.Release() + found := false + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { + if found { + t.Fatal("got more than one file") + } + if i.FileName() != name { + t.Fatalf("got file %v, expected %v", i.FileName(), name) + } + found = true + return true + }) + snap.Release() + + upper := strings.ToUpper(name) + must(t, ffs.Rename(name, upper)) + m.ScanFolders() + + snap = dbSnapshot(t, m, fcfg.ID) + defer snap.Release() + found = false + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { + if i.FileName() == name { + if i.IsDeleted() { + return true + } + t.Fatal("renamed file not deleted") + } + if i.FileName() != upper { + t.Fatalf("got file %v, expected %v", i.FileName(), upper) + } + if found { + t.Fatal("got more than the expected files") + } + found = true + return true + }) +} + func TestConnectionTerminationOnFolderAdd(t *testing.T) { testConfigChangeClosesConnections(t, false, true, nil, func(cfg config.Wrapper) { fcfg := testFolderConfigTmp() diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index 87aba6d34..ee2cf2895 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -323,7 +323,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) { fc.deleteFile(invDel) fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents) fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents) - if err := ioutil.WriteFile(filepath.Join(fss.URI(), ignExisting), otherContents, 0644); err != nil { + if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil { panic(err) } @@ -465,12 +465,12 @@ func TestIssue4841(t *testing.T) { func TestRescanIfHaveInvalidContent(t *testing.T) { m, fc, fcfg := setupModelWithConnection() - tmpDir := fcfg.Filesystem().URI() - defer cleanupModelAndRemoveDir(m, tmpDir) + tfs := fcfg.Filesystem() + defer cleanupModelAndRemoveDir(m, tfs.URI()) payload := []byte("hello") - must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777)) + must(t, writeFile(tfs, "foo", payload, 0777)) received := make(chan []protocol.FileInfo) fc.mut.Lock() @@ -511,7 +511,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) { payload = []byte("bye") buf = make([]byte, len(payload)) - must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777)) + must(t, writeFile(tfs, "foo", payload, 0777)) _, err = m.Request(device1, "default", "foo", int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false) if err == nil { @@ -1051,7 +1051,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) { } fc.mut.Unlock() - if err := ioutil.WriteFile(filepath.Join(fss.URI(), file), contents, 0644); err != nil { + if err := writeFile(fss, file, contents, 0644); err != nil { panic(err) } m.ScanFolders() diff --git a/lib/model/testos_test.go b/lib/model/testos_test.go index 29545fdfc..b1f9e9ee5 100644 --- a/lib/model/testos_test.go +++ b/lib/model/testos_test.go @@ -9,6 +9,8 @@ package model import ( "os" "time" + + "github.com/syncthing/syncthing/lib/fs" ) // fatal is the required common interface between *testing.B and *testing.T @@ -28,6 +30,13 @@ func must(f fatal, err error) { } } +func mustRemove(f fatal, err error) { + f.Helper() + if err != nil && !fs.IsNotExist(err) { + f.Fatal(err) + } +} + func (f *fatalOs) Chmod(name string, mode os.FileMode) { f.Helper() must(f, os.Chmod(name, mode)) diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index 878bb5d52..e4187c281 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -36,9 +36,8 @@ func init() { device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") - defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata") - defaultFolderConfig = testFolderConfig("testdata") + defaultFs = defaultFolderConfig.Filesystem() defaultCfgWrapper = createTmpWrapper(config.New(myID)) _, _ = defaultCfgWrapper.SetDevice(config.NewDeviceConfiguration(device1, "device1")) diff --git a/lib/osutil/osutil.go b/lib/osutil/osutil.go index 2f2973d8b..3d96c7c20 100644 --- a/lib/osutil/osutil.go +++ b/lib/osutil/osutil.go @@ -131,8 +131,10 @@ func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src } func IsDeleted(ffs fs.Filesystem, name string) bool { - if _, err := ffs.Lstat(name); fs.IsNotExist(err) { - return true + if _, err := ffs.Lstat(name); err != nil { + if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) { + return true + } } switch TraversesSymlink(ffs, filepath.Dir(name)).(type) { case *NotADirectoryError, *TraversesSymlinkError: diff --git a/lib/osutil/osutil_test.go b/lib/osutil/osutil_test.go index 861818b56..772097e53 100644 --- a/lib/osutil/osutil_test.go +++ b/lib/osutil/osutil_test.go @@ -62,7 +62,7 @@ func TestIsDeleted(t *testing.T) { } } for _, n := range []string{"Dir", "File", "Del"} { - if err := osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), strings.ToLower(n)), filepath.Join(testFs.URI(), "linkTo"+n)); err != nil { + if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, strings.ToLower(n), "linkTo"+n); err != nil { if runtime.GOOS == "windows" { t.Skip("Symlinks aren't working") } diff --git a/lib/osutil/symlink.go b/lib/osutil/symlink.go deleted file mode 100644 index 4fee84bd6..000000000 --- a/lib/osutil/symlink.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) 2017 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// +build !windows - -package osutil - -import ( - "os" -) - -// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code, -// hence the cumbersome name to make it obvious if this ever leaks. Its -// reason for existence is the Windows version, which allows creating -// symlinks when non-elevated. -func DebugSymlinkForTestsOnly(oldname, newname string) error { - return os.Symlink(oldname, newname) -} diff --git a/lib/osutil/traversessymlink_test.go b/lib/osutil/traversessymlink_test.go index b69affaac..5bf6e7b59 100644 --- a/lib/osutil/traversessymlink_test.go +++ b/lib/osutil/traversessymlink_test.go @@ -24,9 +24,9 @@ func TestTraversesSymlink(t *testing.T) { } defer os.RemoveAll(tmpDir) - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir) - fs.MkdirAll("a/b/c", 0755) - if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(fs.URI(), "a", "b"), filepath.Join(fs.URI(), "a", "l")); err != nil { + testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir) + testFs.MkdirAll("a/b/c", 0755) + if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil { if runtime.GOOS == "windows" { t.Skip("Symlinks aren't working") } @@ -34,7 +34,7 @@ func TestTraversesSymlink(t *testing.T) { } // a/l -> b, so a/l/c should resolve by normal stat - info, err := fs.Lstat("a/l/c") + info, err := testFs.Lstat("a/l/c") if err != nil { t.Fatal("unexpected error", err) } @@ -64,7 +64,7 @@ func TestTraversesSymlink(t *testing.T) { } for _, tc := range cases { - if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) { + if res := osutil.TraversesSymlink(testFs, tc.name); tc.traverses == (res == nil) { t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses) } } @@ -78,8 +78,8 @@ func TestIssue4875(t *testing.T) { defer os.RemoveAll(tmpDir) testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir) - testFs.MkdirAll("a/b/c", 0755) - if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "a", "b"), filepath.Join(testFs.URI(), "a", "l")); err != nil { + testFs.MkdirAll(filepath.Join("a", "b", "c"), 0755) + if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil { if runtime.GOOS == "windows" { t.Skip("Symlinks aren't working") } diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index e180c8e5a..c546ff54c 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -540,6 +540,10 @@ func (w *walker) handleError(ctx context.Context, context, path string, err erro } } +func (w *walker) String() string { + return fmt.Sprintf("walker/%s@%p", w.Folder, w) +} + // A byteCounter gets bytes added to it via Update() and then provides the // Total() and one minute moving average Rate() in bytes per second. type byteCounter struct { diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index b6af968de..2947a481a 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -26,7 +26,6 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" - "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sha256" "golang.org/x/text/unicode/norm" @@ -40,17 +39,19 @@ type testfile struct { type testfileList []testfile -var testFs fs.Filesystem - -var testdata = testfileList{ - {"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"}, - {"dir1", 128, ""}, - {filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"}, - {"dir2", 128, ""}, - {filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"}, - {"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"}, - {"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"}, -} +var ( + testFs fs.Filesystem + testFsType = fs.FilesystemTypeBasic + testdata = testfileList{ + {"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"}, + {"dir1", 128, ""}, + {filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"}, + {"dir2", 128, ""}, + {filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"}, + {"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"}, + {"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"}, + } +) func init() { // This test runs the risk of entering infinite recursion if it fails. @@ -270,7 +271,7 @@ func TestWalkSymlinkUnix(t *testing.T) { defer os.RemoveAll("_symlinks") os.Symlink("../testdata", "_symlinks/link") - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks") + fs := fs.NewFilesystem(testFsType, "_symlinks") for _, path := range []string{".", "link"} { // Scan it files := walkDir(fs, path, nil, nil, 0) @@ -298,15 +299,15 @@ func TestWalkSymlinkWindows(t *testing.T) { os.RemoveAll(name) os.Mkdir(name, 0755) defer os.RemoveAll(name) - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name) - if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil { + testFs := fs.NewFilesystem(testFsType, name) + if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil { // Probably we require permissions we don't have. t.Skip(err) } for _, path := range []string{".", "link"} { // Scan it - files := walkDir(fs, path, nil, nil, 0) + files := walkDir(testFs, path, nil, nil, 0) // Verify that we got zero symlinks if len(files) != 0 { @@ -322,10 +323,12 @@ func TestWalkRootSymlink(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmp) + testFs := fs.NewFilesystem(testFsType, tmp) - link := filepath.Join(tmp, "link") + link := "link" dest, _ := filepath.Abs("testdata/dir1") - if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil { + destFs := fs.NewFilesystem(testFsType, dest) + if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil { if runtime.GOOS == "windows" { // Probably we require permissions we don't have. t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error()) @@ -335,15 +338,15 @@ func TestWalkRootSymlink(t *testing.T) { } // Scan root with symlink at FS root - files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0) + files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0) // Verify that we got two files if len(files) != 2 { - t.Errorf("expected two files, not %d", len(files)) + t.Fatalf("expected two files, not %d", len(files)) } // Scan symlink below FS root - files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), "link", nil, nil, 0) + files = walkDir(testFs, "link", nil, nil, 0) // Verify that we got the one symlink, except on windows if runtime.GOOS == "windows" { @@ -355,7 +358,7 @@ func TestWalkRootSymlink(t *testing.T) { } // Scan path below symlink - files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), filepath.Join("link", "cfile"), nil, nil, 0) + files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0) // Verify that we get nothing if len(files) != 0 { @@ -554,7 +557,7 @@ func BenchmarkHashFile(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil { + if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil { b.Fatal(err) } } @@ -652,7 +655,7 @@ func TestIssue4799(t *testing.T) { } defer os.RemoveAll(tmp) - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp) + fs := fs.NewFilesystem(testFsType, tmp) fd, err := fs.Create("foo") if err != nil { @@ -714,7 +717,7 @@ func TestIssue4841(t *testing.T) { } defer os.RemoveAll(tmp) - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp) + fs := fs.NewFilesystem(testFsType, tmp) fd, err := fs.Create("foo") if err != nil { diff --git a/lib/ur/contract/contract.go b/lib/ur/contract/contract.go index ff503675b..441c68f54 100644 --- a/lib/ur/contract/contract.go +++ b/lib/ur/contract/contract.go @@ -128,6 +128,7 @@ type Report struct { DisableFsync int `json:"disableFsync,omitempty" since:"3"` BlockPullOrder map[string]int `json:"blockPullOrder,omitempty" since:"3"` CopyRangeMethod map[string]int `json:"copyRangeMethod,omitempty" since:"3"` + CaseSensitiveFS int `json:"caseSensitiveFS,omitempty" since:"3"` } `json:"folderUsesV3,omitempty" since:"3"` GUIStats struct { diff --git a/lib/ur/usage_report.go b/lib/ur/usage_report.go index 1ed5d7c8f..3fbf794f0 100644 --- a/lib/ur/usage_report.go +++ b/lib/ur/usage_report.go @@ -269,6 +269,9 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) ( } report.FolderUsesV3.BlockPullOrder[cfg.BlockPullOrder.String()]++ report.FolderUsesV3.CopyRangeMethod[cfg.CopyRangeMethod.String()]++ + if cfg.CaseSensitiveFS { + report.FolderUsesV3.CaseSensitiveFS++ + } } sort.Ints(report.FolderUsesV3.FsWatcherDelays) diff --git a/test/transfer-bench_test.go b/test/transfer-bench_test.go index 53bafaf70..70e26a722 100644 --- a/test/transfer-bench_test.go +++ b/test/transfer-bench_test.go @@ -10,42 +10,42 @@ package integration import ( "log" + "math/rand" "os" "testing" "time" + + "github.com/syncthing/syncthing/lib/rc" ) func TestBenchmarkTransferManyFiles(t *testing.T) { - benchmarkTransfer(t, 10000, 15) + setupAndBenchmarkTransfer(t, 10000, 15) } func TestBenchmarkTransferLargeFile1G(t *testing.T) { - benchmarkTransfer(t, 1, 30) + setupAndBenchmarkTransfer(t, 1, 30) } func TestBenchmarkTransferLargeFile2G(t *testing.T) { - benchmarkTransfer(t, 1, 31) + setupAndBenchmarkTransfer(t, 1, 31) } func TestBenchmarkTransferLargeFile4G(t *testing.T) { - benchmarkTransfer(t, 1, 32) + setupAndBenchmarkTransfer(t, 1, 32) } func TestBenchmarkTransferLargeFile8G(t *testing.T) { - benchmarkTransfer(t, 1, 33) + setupAndBenchmarkTransfer(t, 1, 33) } func TestBenchmarkTransferLargeFile16G(t *testing.T) { - benchmarkTransfer(t, 1, 34) + setupAndBenchmarkTransfer(t, 1, 34) } func TestBenchmarkTransferLargeFile32G(t *testing.T) { - benchmarkTransfer(t, 1, 35) + setupAndBenchmarkTransfer(t, 1, 35) } -func benchmarkTransfer(t *testing.T, files, sizeExp int) { - log.Println("Cleaning...") - err := removeAll("s1", "s2", "h1/index*", "h2/index*") - if err != nil { - t.Fatal(err) - } +func setupAndBenchmarkTransfer(t *testing.T, files, sizeExp int) { + cleanBenchmarkTransfer(t) log.Println("Generating files...") + var err error if files == 1 { // Special case. Generate one file with the specified size exactly. var fd *os.File @@ -57,13 +57,39 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) { if err != nil { t.Fatal(err) } - err = generateOneFile(fd, "s1/onefile", 1<