Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708) 

With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei 2020-07-28 11:13:15 +02:00 committed by Jakob Borg
parent 21dd9d6b43
commit 932d8c69de
27 changed files with 1241 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/ursrv/main.go
View File

@ -743,6 +743,7 @@ func getReport(db *sql.DB) map[string]interface{} {
inc(features["Folder"]["v3"], "Weak hash, always", rep.FolderUsesV3.AlwaysWeakHash)
inc(features["Folder"]["v3"], "Weak hash, custom threshold", rep.FolderUsesV3.CustomWeakHashThreshold)
inc(features["Folder"]["v3"], "Filesystem watcher", rep.FolderUsesV3.FsWatcherEnabled)
inc(features["Folder"]["v3"], "Case sensitive FS", rep.FolderUsesV3.CaseSensitiveFS)
add(featureGroups["Folder"]["v3"], "Conflicts", "Disabled", rep.FolderUsesV3.ConflictsDisabled)
add(featureGroups["Folder"]["v3"], "Conflicts", "Unlimited", rep.FolderUsesV3.ConflictsUnlimited)

11
lib/config/config_test.go
View File

@ -22,7 +22,6 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
)
@ -129,13 +128,6 @@ func TestDeviceConfig(t *testing.T) {
},
}
// The cachedFilesystem will have been resolved to an absolute path,
// depending on where the tests are running. Zero it out so we don't
// fail based on that.
for i := range cfg.Folders {
cfg.Folders[i].cachedFilesystem = nil
}
expectedDevices := []DeviceConfiguration{
{
DeviceID: device1,
@ -465,6 +457,7 @@ func TestFolderCheckPath(t *testing.T) {
if err != nil {
t.Fatal(err)
}
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, n)
err = os.MkdirAll(filepath.Join(n, "dir", ".stfolder"), os.FileMode(0777))
if err != nil {
@ -489,7 +482,7 @@ func TestFolderCheckPath(t *testing.T) {
},
}
err = osutil.DebugSymlinkForTestsOnly(filepath.Join(n, "dir"), filepath.Join(n, "link"))
err = fs.DebugSymlinkForTestsOnly(testFs, testFs, "dir", "link")
if err == nil {
t.Log("running with symlink check")
testcases = append(testcases, struct {

13
lib/config/folderconfiguration.go
View File

@ -61,8 +61,8 @@ type FolderConfiguration struct {
DisableFsync bool `xml:"disableFsync" json:"disableFsync"`
BlockPullOrder BlockPullOrder `xml:"blockPullOrder" json:"blockPullOrder"`
CopyRangeMethod fs.CopyRangeMethod `xml:"copyRangeMethod" json:"copyRangeMethod" default:"standard"`
CaseSensitiveFS bool `xml:"caseSensitiveFS" json:"caseSensitiveFS"`
cachedFilesystem fs.Filesystem
cachedModTimeWindow time.Duration
DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
@ -101,11 +101,11 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
func (f FolderConfiguration) Filesystem() fs.Filesystem {
// This is intentionally not a pointer method, because things like
// cfg.Folders["default"].Filesystem() should be valid.
if f.cachedFilesystem == nil {
l.Infoln("bug: uncached filesystem call (should only happen in tests)")
return fs.NewFilesystem(f.FilesystemType, f.Path)
filesystem := fs.NewFilesystem(f.FilesystemType, f.Path)
if !f.CaseSensitiveFS {
filesystem = fs.NewCaseFilesystem(filesystem)
}
return f.cachedFilesystem
return filesystem
}
func (f FolderConfiguration) ModTimeWindow() time.Duration {
@ -210,8 +210,6 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
}
func (f *FolderConfiguration) prepare() {
f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
if f.RescanIntervalS > MaxRescanIntervalS {
f.RescanIntervalS = MaxRescanIntervalS
} else if f.RescanIntervalS < 0 {
@ -263,7 +261,6 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
// Manual handling for things that are not taken care of by the tag
// copier, yet should not cause a restart.
copy.cachedFilesystem = nil
blank := FolderConfiguration{}
util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {

13
lib/fs/basicfs_realcaser_unix.go Normal file
View File

@ -0,0 +1,13 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
func newBasicRealCaser(fs Filesystem) realCaser {
return newDefaultRealCaser(fs)
}

58
lib/fs/basicfs_realcaser_windows.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build windows
package fs
import (
"path/filepath"
"strings"
"syscall"
)
type basicRealCaserWindows struct {
uri string
}
func newBasicRealCaser(fs Filesystem) realCaser {
return &basicRealCaserWindows{fs.URI()}
}
// RealCase returns the correct case for the given name, which is a relative
// path below root, as it exists on disk.
func (r *basicRealCaserWindows) realCase(name string) (string, error) {
if name == "." {
return ".", nil
}
path := r.uri
comps := strings.Split(name, string(PathSeparator))
var err error
for i, comp := range comps {
path = filepath.Join(path, comp)
comps[i], err = r.realCaseBase(path)
if err != nil {
return "", err
}
}
return filepath.Join(comps...), nil
}
func (*basicRealCaserWindows) realCaseBase(path string) (string, error) {
p, err := syscall.UTF16PtrFromString(fixLongPath(path))
if err != nil {
return "", err
}
var fd syscall.Win32finddata
h, err := syscall.FindFirstFile(p, &fd)
if err != nil {
return "", err
}
syscall.FindClose(h)
return syscall.UTF16ToString(fd.FileName[:]), nil
}
func (r *basicRealCaserWindows) dropCache() {}

448
lib/fs/casefs.go Normal file
View File

@ -0,0 +1,448 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"sync"
"time"
)
// Both values were chosen by magic.
const (
caseCacheTimeout = time.Second
// When the number of names (all lengths of []string from DirNames)
// exceeds this, we drop the cache.
caseMaxCachedNames = 1 << 20
)
type ErrCaseConflict struct {
given, real string
}
func (e *ErrCaseConflict) Error() string {
return fmt.Sprintf(`given name "%v" differs from name in filesystem "%v"`, e.given, e.real)
}
func IsErrCaseConflict(err error) bool {
e := &ErrCaseConflict{}
return errors.As(err, &e)
}
type realCaser interface {
realCase(name string) (string, error)
dropCache()
}
type fskey struct {
fstype FilesystemType
uri string
}
var (
caseFilesystems = make(map[fskey]Filesystem)
caseFilesystemsMut sync.Mutex
)
// caseFilesystem is a BasicFilesystem with additional checks to make a
// potentially case insensitive underlying FS behave like it's case-sensitive.
type caseFilesystem struct {
Filesystem
realCaser
}
// NewCaseFilesystem ensures that the given, potentially case-insensitive filesystem
// behaves like a case-sensitive filesystem. Meaning that it takes into account
// the real casing of a path and returns ErrCaseConflict if the given path differs
// from the real path. It is safe to use with any filesystem, i.e. also a
// case-sensitive one. However it will add some overhead and thus shouldn't be
// used if the filesystem is known to already behave case-sensitively.
func NewCaseFilesystem(fs Filesystem) Filesystem {
caseFilesystemsMut.Lock()
defer caseFilesystemsMut.Unlock()
k := fskey{fs.Type(), fs.URI()}
if caseFs, ok := caseFilesystems[k]; ok {
return caseFs
}
caseFs := &caseFilesystem{
Filesystem: fs,
}
switch k.fstype {
case FilesystemTypeBasic:
caseFs.realCaser = newBasicRealCaser(fs)
default:
caseFs.realCaser = newDefaultRealCaser(fs)
}
caseFilesystems[k] = caseFs
return caseFs
}
func (f *caseFilesystem) Chmod(name string, mode FileMode) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Chmod(name, mode)
}
func (f *caseFilesystem) Lchown(name string, uid, gid int) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Lchown(name, uid, gid)
}
func (f *caseFilesystem) Chtimes(name string, atime time.Time, mtime time.Time) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Chtimes(name, atime, mtime)
}
func (f *caseFilesystem) Mkdir(name string, perm FileMode) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.Mkdir(name, perm); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) MkdirAll(path string, perm FileMode) error {
if err := f.checkCase(path); err != nil {
return err
}
if err := f.Filesystem.MkdirAll(path, perm); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Lstat(name string) (FileInfo, error) {
var err error
if name, err = Canonicalize(name); err != nil {
return nil, err
}
stat, err := f.Filesystem.Lstat(name)
if err != nil {
return nil, err
}
if err = f.checkCaseExisting(name); err != nil {
return nil, err
}
return stat, nil
}
func (f *caseFilesystem) Remove(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.Remove(name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) RemoveAll(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.RemoveAll(name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Rename(oldpath, newpath string) error {
if err := f.checkCase(oldpath); err != nil {
return err
}
if err := f.Filesystem.Rename(oldpath, newpath); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Stat(name string) (FileInfo, error) {
var err error
if name, err = Canonicalize(name); err != nil {
return nil, err
}
stat, err := f.Filesystem.Stat(name)
if err != nil {
return nil, err
}
if err = f.checkCaseExisting(name); err != nil {
return nil, err
}
return stat, nil
}
func (f *caseFilesystem) DirNames(name string) ([]string, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
return f.Filesystem.DirNames(name)
}
func (f *caseFilesystem) Open(name string) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
return f.Filesystem.Open(name)
}
func (f *caseFilesystem) OpenFile(name string, flags int, mode FileMode) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
file, err := f.Filesystem.OpenFile(name, flags, mode)
if err != nil {
return nil, err
}
f.dropCache()
return file, nil
}
func (f *caseFilesystem) ReadSymlink(name string) (string, error) {
if err := f.checkCase(name); err != nil {
return "", err
}
return f.Filesystem.ReadSymlink(name)
}
func (f *caseFilesystem) Create(name string) (File, error) {
if err := f.checkCase(name); err != nil {
return nil, err
}
file, err := f.Filesystem.Create(name)
if err != nil {
return nil, err
}
f.dropCache()
return file, nil
}
func (f *caseFilesystem) CreateSymlink(target, name string) error {
if err := f.checkCase(name); err != nil {
return err
}
if err := f.Filesystem.CreateSymlink(target, name); err != nil {
return err
}
f.dropCache()
return nil
}
func (f *caseFilesystem) Walk(root string, walkFn WalkFunc) error {
// Walking the filesystem is likely (in Syncthing's case certainly) done
// to pick up external changes, for which caching is undesirable.
f.dropCache()
if err := f.checkCase(root); err != nil {
return err
}
return f.Filesystem.Walk(root, walkFn)
}
func (f *caseFilesystem) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
if err := f.checkCase(path); err != nil {
return nil, nil, err
}
return f.Filesystem.Watch(path, ignore, ctx, ignorePerms)
}
func (f *caseFilesystem) Hide(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Hide(name)
}
func (f *caseFilesystem) Unhide(name string) error {
if err := f.checkCase(name); err != nil {
return err
}
return f.Filesystem.Unhide(name)
}
func (f *caseFilesystem) checkCase(name string) error {
var err error
if name, err = Canonicalize(name); err != nil {
return err
}
// Stat is necessary for case sensitive FS, as it's then not a conflict
// if name is e.g. "foo" and on dir there is "Foo".
if _, err := f.Filesystem.Lstat(name); err != nil {
if IsNotExist(err) {
return nil
}
return err
}
return f.checkCaseExisting(name)
}
// checkCaseExisting must only be called after successfully canonicalizing and
// stating the file.
func (f *caseFilesystem) checkCaseExisting(name string) error {
realName, err := f.realCase(name)
if IsNotExist(err) {
// It did exist just before -> cache is outdated, try again
f.dropCache()
realName, err = f.realCase(name)
}
if err != nil {
return err
}
if realName != name {
return &ErrCaseConflict{name, realName}
}
return nil
}
type defaultRealCaser struct {
fs Filesystem
root *caseNode
count int
timer *time.Timer
timerStop chan struct{}
mut sync.RWMutex
}
func newDefaultRealCaser(fs Filesystem) *defaultRealCaser {
caser := &defaultRealCaser{
fs: fs,
root: &caseNode{name: "."},
timer: time.NewTimer(0),
}
<-caser.timer.C
return caser
}
func (r *defaultRealCaser) realCase(name string) (string, error) {
out := "."
if name == out {
return out, nil
}
r.mut.Lock()
defer func() {
if r.count > caseMaxCachedNames {
select {
case r.timerStop <- struct{}{}:
default:
}
r.dropCacheLocked()
}
r.mut.Unlock()
}()
node := r.root
for _, comp := range strings.Split(name, string(PathSeparator)) {
if node.dirNames == nil {
// Haven't called DirNames yet
var err error
node.dirNames, err = r.fs.DirNames(out)
if err != nil {
return "", err
}
node.dirNamesLower = make([]string, len(node.dirNames))
for i, n := range node.dirNames {
node.dirNamesLower[i] = UnicodeLowercase(n)
}
node.children = make(map[string]*caseNode)
node.results = make(map[string]*caseNode)
r.count += len(node.dirNames)
} else if child, ok := node.results[comp]; ok {
// Check if this exact name has been queried before to shortcut
node = child
out = filepath.Join(out, child.name)
continue
}
// Actually loop dirNames to search for a match
n, err := findCaseInsensitiveMatch(comp, node.dirNames, node.dirNamesLower)
if err != nil {
return "", err
}
child, ok := node.children[n]
if !ok {
child = &caseNode{name: n}
}
node.results[comp] = child
node.children[n] = child
node = child
out = filepath.Join(out, n)
}
return out, nil
}
func (r *defaultRealCaser) startCaseResetTimerLocked() {
r.timerStop = make(chan struct{})
r.timer.Reset(caseCacheTimeout)
go func() {
select {
case <-r.timer.C:
r.dropCache()
case <-r.timerStop:
if !r.timer.Stop() {
<-r.timer.C
}
r.mut.Lock()
r.timerStop = nil
r.mut.Unlock()
}
}()
}
func (r *defaultRealCaser) dropCache() {
r.mut.Lock()
r.dropCacheLocked()
r.mut.Unlock()
}
func (r *defaultRealCaser) dropCacheLocked() {
r.root = &caseNode{name: "."}
r.count = 0
}
// Both name and the key to children are "Real", case resolved names of the path
// component this node represents (i.e. containing no path separator).
// The key to results is also a path component, but as given to RealCase, not
// case resolved.
type caseNode struct {
name string
dirNames []string
dirNamesLower []string
children map[string]*caseNode
results map[string]*caseNode
}
func findCaseInsensitiveMatch(name string, names, namesLower []string) (string, error) {
lower := UnicodeLowercase(name)
candidate := ""
for i, n := range names {
if n == name {
return n, nil
}
if candidate == "" && namesLower[i] == lower {
candidate = n
}
}
if candidate == "" {
return "", ErrNotExist
}
return candidate, nil
}

279
lib/fs/casefs_test.go Normal file
View File

@ -0,0 +1,279 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
func TestRealCase(t *testing.T) {
// Verify realCase lookups on various underlying filesystems.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testRealCase(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCase(t, fsys)
})
}
func testRealCase(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
comps := []string{"Foo", "bar", "BAZ", "bAs"}
path := filepath.Join(comps...)
testFs.MkdirAll(filepath.Join(comps[:len(comps)-1]...), 0777)
fd, err := testFs.Create(path)
if err != nil {
t.Fatal(err)
}
fd.Close()
for i, tc := range []struct {
in string
len int
}{
{path, 4},
{strings.ToLower(path), 4},
{strings.ToUpper(path), 4},
{"foo", 1},
{"FOO", 1},
{"foO", 1},
{filepath.Join("Foo", "bar"), 2},
{filepath.Join("Foo", "bAr"), 2},
{filepath.Join("FoO", "bar"), 2},
{filepath.Join("foo", "bar", "BAZ"), 3},
{filepath.Join("Foo", "bar", "bAz"), 3},
{filepath.Join("foo", "bar", "BAZ"), 3}, // Repeat on purpose
} {
out, err := testFs.realCase(tc.in)
if err != nil {
t.Error(err)
} else if exp := filepath.Join(comps[:tc.len]...); out != exp {
t.Errorf("tc %v: Expected %v, got %v", i, exp, out)
}
}
}
func TestRealCaseSensitive(t *testing.T) {
// Verify that realCase returns the best on-disk case for case sensitive
// systems. Test is skipped if the underlying fs is insensitive.
t.Run("fake-sensitive", func(t *testing.T) {
testRealCaseSensitive(t, newFakeFilesystem(t.Name()))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testRealCaseSensitive(t, fsys)
})
}
func testRealCaseSensitive(t *testing.T, fsys Filesystem) {
testFs := NewCaseFilesystem(fsys).(*caseFilesystem)
names := make([]string, 2)
names[0] = "foo"
names[1] = strings.ToUpper(names[0])
for _, n := range names {
if err := testFs.MkdirAll(n, 0777); err != nil {
if IsErrCaseConflict(err) {
t.Skip("Filesystem is case-insensitive")
}
t.Fatal(err)
}
}
for _, n := range names {
if rn, err := testFs.realCase(n); err != nil {
t.Error(err)
} else if rn != n {
t.Errorf("Got %v, expected %v", rn, n)
}
}
}
func TestCaseFSStat(t *testing.T) {
// Verify that a Stat() lookup behaves in a case sensitive manner
// regardless of the underlying fs.
t.Run("fake-sensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()))
})
t.Run("fake-insensitive", func(t *testing.T) {
testCaseFSStat(t, newFakeFilesystem(t.Name()+"?insens=true"))
})
t.Run("actual", func(t *testing.T) {
fsys, tmpDir := setup(t)
defer os.RemoveAll(tmpDir)
testCaseFSStat(t, fsys)
})
}
func testCaseFSStat(t *testing.T, fsys Filesystem) {
fd, err := fsys.Create("foo")
if err != nil {
t.Fatal(err)
}
fd.Close()
// Check if the underlying fs is sensitive or not
sensitive := true
if _, err = fsys.Stat("FOO"); err == nil {
sensitive = false
}
testFs := NewCaseFilesystem(fsys)
_, err = testFs.Stat("FOO")
if sensitive {
if IsNotExist(err) {
t.Log("pass: case sensitive underlying fs")
} else {
t.Error("expected NotExist, not", err, "for sensitive fs")
}
} else if IsErrCaseConflict(err) {
t.Log("pass: case insensitive underlying fs")
} else {
t.Error("expected ErrCaseConflict, not", err, "for insensitive fs")
}
}
func BenchmarkWalkCaseFakeFS10k(b *testing.B) {
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
b.Fatal(err)
}
slowsys, paths, err := fakefsForBenchmark(10_000, 100*time.Microsecond)
if err != nil {
b.Fatal(err)
}
b.Run("raw-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, fsys, paths)
b.ReportAllocs()
})
b.Run("case-fastfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(fsys), paths)
b.ReportAllocs()
})
b.Run("raw-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, slowsys, paths)
b.ReportAllocs()
})
b.Run("case-slowfs", func(b *testing.B) {
benchmarkWalkFakeFS(b, NewCaseFilesystem(slowsys), paths)
b.ReportAllocs()
})
}
func benchmarkWalkFakeFS(b *testing.B, fsys Filesystem, paths []string) {
// Simulate a scanner pass over the filesystem. First walk it to
// discover all names, then stat each name individually to check if it's
// been deleted or not (pretending that they all existed in the
// database).
var ms0 runtime.MemStats
runtime.ReadMemStats(&ms0)
t0 := time.Now()
for i := 0; i < b.N; i++ {
if err := doubleWalkFS(fsys, paths); err != nil {
b.Fatal(err)
}
}
t1 := time.Now()
var ms1 runtime.MemStats
runtime.ReadMemStats(&ms1)
// We add metrics per path entry
b.ReportMetric(float64(t1.Sub(t0))/float64(b.N)/float64(len(paths)), "ns/entry")
b.ReportMetric(float64(ms1.Mallocs-ms0.Mallocs)/float64(b.N)/float64(len(paths)), "allocs/entry")
b.ReportMetric(float64(ms1.TotalAlloc-ms0.TotalAlloc)/float64(b.N)/float64(len(paths)), "B/entry")
}
func TestStressCaseFS(t *testing.T) {
// Exercise a bunch of paralell operations for stressing out race
// conditions in the realnamer cache etc.
const limit = 10 * time.Second
if testing.Short() {
t.Skip("long test")
}
fsys, paths, err := fakefsForBenchmark(10_000, 0)
if err != nil {
t.Fatal(err)
}
for i := 0; i < runtime.NumCPU()/2+1; i++ {
t.Run(fmt.Sprintf("walker-%d", i), func(t *testing.T) {
// Walk the filesystem and stat everything
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
if err := doubleWalkFS(fsys, paths); err != nil {
t.Fatal(err)
}
}
})
t.Run(fmt.Sprintf("toucher-%d", i), func(t *testing.T) {
// Touch all the things
t.Parallel()
t0 := time.Now()
for time.Since(t0) < limit {
for _, p := range paths {
now := time.Now()
if err := fsys.Chtimes(p, now, now); err != nil {
t.Fatal(err)
}
}
}
})
}
}
func doubleWalkFS(fsys Filesystem, paths []string) error {
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
return err
}); err != nil {
return err
}
for _, p := range paths {
if _, err := fsys.Lstat(p); err != nil {
return err
}
}
return nil
}
func fakefsForBenchmark(nfiles int, latency time.Duration) (Filesystem, []string, error) {
fsys := NewFilesystem(FilesystemTypeFake, fmt.Sprintf("fakefsForBenchmark?files=%d&insens=true&latency=%s", nfiles, latency))
var paths []string
if err := fsys.Walk("/", func(path string, info FileInfo, err error) error {
paths = append(paths, path)
return err
}); err != nil {
return nil, nil, err
}
if len(paths) < nfiles {
return nil, nil, errors.New("didn't find enough stuff")
}
return fsys, paths, nil
}

47
lib/fs/debug_symlink_unix.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package fs
import (
"os"
"path/filepath"
)
// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
// hence the cumbersome name to make it obvious if this ever leaks. Its
// reason for existence is the Windows version, which allows creating
// symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldFs, newFs Filesystem, oldname, newname string) error {
if caseFs, ok := unwrapFilesystem(newFs).(*caseFilesystem); ok {
if err := caseFs.checkCase(newname); err != nil {
return err
}
caseFs.dropCache()
}
if err := os.Symlink(filepath.Join(oldFs.URI(), oldname), filepath.Join(newFs.URI(), newname)); err != nil {
return err
}
return nil
}
// unwrapFilesystem removes "wrapping" filesystems to expose the underlying filesystem.
func unwrapFilesystem(fs Filesystem) Filesystem {
for {
switch sfs := fs.(type) {
case *logFilesystem:
fs = sfs.Filesystem
case *walkFilesystem:
fs = sfs.Filesystem
case *MtimeFS:
fs = sfs.Filesystem
default:
return sfs
}
}
}

7
lib/osutil/symlink_windows.go → lib/fs/debug_symlink_windows.go
View File

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package osutil
package fs
import (
"os"
@ -17,7 +17,10 @@ import (
// This is not and should not be used in Syncthing code, hence the
// cumbersome name to make it obvious if this ever leaks. Nonetheless it's
// useful in tests.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
func DebugSymlinkForTestsOnly(oldFs, newFS Filesystem, oldname, newname string) error {
oldname = filepath.Join(oldFs.URI(), oldname)
newname = filepath.Join(newFS.URI(), newname)
// CreateSymbolicLink is not supported before Windows Vista
if syscall.LoadCreateSymbolicLink() != nil {
return &os.LinkError{"symlink", oldname, newname, syscall.EWINDOWS}

34
lib/fs/fakefs.go
View File

@ -48,14 +48,17 @@ const randomBlockShift = 14 // 128k
// sizeavg=n to set the average size of random files, in bytes (default 1<<20)
// seed=n to set the initial random seed (default 0)
// insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
// latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
//
// - Two fakefs:s pointing at the same root path see the same files.
//
type fakefs struct {
uri string
mut sync.Mutex
root *fakeEntry
insens bool
withContent bool
latency time.Duration
}
var (
@ -63,23 +66,25 @@ var (
fakefsFs = make(map[string]*fakefs)
)
func newFakeFilesystem(root string) *fakefs {
func newFakeFilesystem(rootURI string) *fakefs {
fakefsMut.Lock()
defer fakefsMut.Unlock()
root := rootURI
var params url.Values
uri, err := url.Parse(root)
uri, err := url.Parse(rootURI)
if err == nil {
root = uri.Path
params = uri.Query()
}
if fs, ok := fakefsFs[root]; ok {
if fs, ok := fakefsFs[rootURI]; ok {
// Already have an fs at this path
return fs
}
fs := &fakefs{
uri: "fake://" + rootURI,
root: &fakeEntry{
name: "/",
entryType: fakeEntryTypeDir,
@ -129,6 +134,10 @@ func newFakeFilesystem(root string) *fakefs {
// Also create a default folder marker for good measure
fs.Mkdir(".stfolder", 0700)
// We only set the latency after doing the operations required to create
// the filesystem initially.
fs.latency, _ = time.ParseDuration(params.Get("latency"))
fakefsFs[root] = fs
return fs
}
@ -185,6 +194,7 @@ func (fs *fakefs) entryForName(name string) *fakeEntry {
func (fs *fakefs) Chmod(name string, mode FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -196,6 +206,7 @@ func (fs *fakefs) Chmod(name string, mode FileMode) error {
func (fs *fakefs) Lchown(name string, uid, gid int) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -208,6 +219,7 @@ func (fs *fakefs) Lchown(name string, uid, gid int) error {
func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
return os.ErrNotExist
@ -219,6 +231,7 @@ func (fs *fakefs) Chtimes(name string, atime time.Time, mtime time.Time) error {
func (fs *fakefs) create(name string) (*fakeEntry, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if entry := fs.entryForName(name); entry != nil {
if entry.entryType == fakeEntryTypeDir {
@ -284,6 +297,7 @@ func (fs *fakefs) CreateSymlink(target, name string) error {
func (fs *fakefs) DirNames(name string) ([]string, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -301,6 +315,7 @@ func (fs *fakefs) DirNames(name string) ([]string, error) {
func (fs *fakefs) Lstat(name string) (FileInfo, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -318,6 +333,7 @@ func (fs *fakefs) Lstat(name string) (FileInfo, error) {
func (fs *fakefs) Mkdir(name string, perm FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
dir := filepath.Dir(name)
base := filepath.Base(name)
@ -348,6 +364,10 @@ func (fs *fakefs) Mkdir(name string, perm FileMode) error {
}
func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
name = filepath.ToSlash(name)
name = strings.Trim(name, "/")
comps := strings.Split(name, "/")
@ -382,6 +402,7 @@ func (fs *fakefs) MkdirAll(name string, perm FileMode) error {
func (fs *fakefs) Open(name string) (File, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil || entry.entryType != fakeEntryTypeFile {
@ -401,6 +422,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
dir := filepath.Dir(name)
base := filepath.Base(name)
@ -438,6 +460,7 @@ func (fs *fakefs) OpenFile(name string, flags int, mode FileMode) (File, error)
func (fs *fakefs) ReadSymlink(name string) (string, error) {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
entry := fs.entryForName(name)
if entry == nil {
@ -451,6 +474,7 @@ func (fs *fakefs) ReadSymlink(name string) (string, error) {
func (fs *fakefs) Remove(name string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if fs.insens {
name = UnicodeLowercase(name)
@ -472,6 +496,7 @@ func (fs *fakefs) Remove(name string) error {
func (fs *fakefs) RemoveAll(name string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
if fs.insens {
name = UnicodeLowercase(name)
@ -491,6 +516,7 @@ func (fs *fakefs) RemoveAll(name string) error {
func (fs *fakefs) Rename(oldname, newname string) error {
fs.mut.Lock()
defer fs.mut.Unlock()
time.Sleep(fs.latency)
oldKey := filepath.Base(oldname)
newKey := filepath.Base(newname)
@ -578,7 +604,7 @@ func (fs *fakefs) Type() FilesystemType {
}
func (fs *fakefs) URI() string {
return "fake://" + fs.root.name
return fs.uri
}
func (fs *fakefs) SameFile(fi1, fi2 FileInfo) bool {

41
lib/model/folder_sendrecv.go
View File

@ -129,9 +129,9 @@ type sendReceiveFolder struct {
blockPullReorderer blockPullReorderer
writeLimiter *byteSemaphore
pullErrors map[string]string // errors for most recent/current iteration
oldPullErrors map[string]string // errors from previous iterations for log filtering only
pullErrorsMut sync.Mutex
pullErrors map[string]string // actual exposed pull errors
tempPullErrors map[string]string // pull errors that might be just transient
pullErrorsMut sync.Mutex
}
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger, ioLimiter *byteSemaphore) service {
@ -192,6 +192,10 @@ func (f *sendReceiveFolder) pull() bool {
changed := 0
f.pullErrorsMut.Lock()
f.pullErrors = nil
f.pullErrorsMut.Unlock()
for tries := 0; tries < maxPullerIterations; tries++ {
select {
case <-f.ctx.Done():
@ -216,8 +220,14 @@ func (f *sendReceiveFolder) pull() bool {
}
f.pullErrorsMut.Lock()
f.pullErrors = f.tempPullErrors
f.tempPullErrors = nil
for path, err := range f.pullErrors {
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
}
pullErrNum := len(f.pullErrors)
f.pullErrorsMut.Unlock()
if pullErrNum > 0 {
l.Infof("%v: Failed to sync %v items", f.Description(), pullErrNum)
f.evLogger.Log(events.FolderErrors, map[string]interface{}{
@ -235,8 +245,7 @@ func (f *sendReceiveFolder) pull() bool {
// flagged as needed in the folder.
func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
f.pullErrorsMut.Lock()
f.oldPullErrors = f.pullErrors
f.pullErrors = make(map[string]string)
f.tempPullErrors = make(map[string]string)
f.pullErrorsMut.Unlock()
snap := f.fset.Snapshot()
@ -306,10 +315,6 @@ func (f *sendReceiveFolder) pullerIteration(scanChan chan<- string) int {
close(dbUpdateChan)
updateWg.Wait()
f.pullErrorsMut.Lock()
f.oldPullErrors = nil
f.pullErrorsMut.Unlock()
f.queue.Reset()
return changed
@ -739,7 +744,11 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, snap *db.Snaps
}
// There is already something under that name, we need to handle that.
if info, err := f.fs.Lstat(file.Name); err == nil {
switch info, err := f.fs.Lstat(file.Name); {
case err != nil && !fs.IsNotExist(err):
f.newPullError(file.Name, errors.Wrap(err, "checking for existing symlink"))
return
case err == nil:
// Check that it is what we have in the database.
curFile, hasCurFile := f.model.CurrentFolderFile(f.folderID, file.Name)
if err := f.scanIfItemChanged(file.Name, info, curFile, hasCurFile, scanChan); err != nil {
@ -1783,7 +1792,7 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
// We might get more than one error report for a file (i.e. error on
// Write() followed by Close()); we keep the first error as that is
// probably closer to the root cause.
if _, ok := f.pullErrors[path]; ok {
if _, ok := f.tempPullErrors[path]; ok {
return
}
@ -1791,15 +1800,9 @@ func (f *sendReceiveFolder) newPullError(path string, err error) {
// Use "syncing" as opposed to "pulling" as the latter might be used
// for errors occurring specificly in the puller routine.
errStr := fmt.Sprintln("syncing:", err)
f.pullErrors[path] = errStr
f.tempPullErrors[path] = errStr
if oldErr, ok := f.oldPullErrors[path]; ok && oldErr == errStr {
l.Debugf("Repeat error on puller (folder %s, item %q): %v", f.Description(), path, err)
delete(f.oldPullErrors, path) // Potential repeats are now caught by f.pullErrors itself
return
}
l.Infof("Puller (folder %s, item %q): %v", f.Description(), path, err)
l.Debugf("%v new error for %v: %v", f, path, err)
}
func (f *sendReceiveFolder) Errors() []FileError {

121
lib/model/folder_sendrecv_test.go
View File

@ -10,11 +10,13 @@ import (
"bytes"
"context"
"crypto/rand"
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
@ -95,6 +97,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
model.Supervisor.Stop()
f := model.folderRunners[fcfg.ID].(*sendReceiveFolder)
f.pullErrors = make(map[string]string)
f.tempPullErrors = make(map[string]string)
f.ctx = context.Background()
// Update index
@ -983,7 +986,7 @@ func TestDeleteBehindSymlink(t *testing.T) {
must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file"))
must(t, ffs.RemoveAll(link))
if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil {
if err := fs.DebugSymlinkForTestsOnly(destFs, ffs, "", link); err != nil {
if runtime.GOOS == "windows" {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@ -1087,6 +1090,122 @@ func TestPullDeleteUnscannedDir(t *testing.T) {
}
}
func TestPullCaseOnlyPerformFinish(t *testing.T) {
m, f := setupSendReceiveFolder()
defer cleanupSRFolder(f, m)
ffs := f.Filesystem()
name := "foo"
contents := []byte("contents")
must(t, writeFile(ffs, name, contents, 0644))
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if hasCur {
t.Fatal("got more than one file")
}
cur = i.(protocol.FileInfo)
hasCur = true
return true
})
if !hasCur {
t.Fatal("file is missing")
}
remote := *(&cur)
remote.Version = protocol.Vector{}.Update(device1.Short())
remote.Name = strings.ToUpper(cur.Name)
temp := fs.TempName(remote.Name)
must(t, writeFile(ffs, temp, contents, 0644))
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
err := f.performFinish(remote, cur, hasCur, temp, snap, dbUpdateChan, scanChan)
select {
case <-dbUpdateChan: // boring case sensitive filesystem
return
case <-scanChan:
t.Error("no need to scan anything here")
default:
}
var caseErr *fs.ErrCaseConflict
if !errors.As(err, &caseErr) {
t.Error("Expected case conflict error, got", err)
}
}
func TestPullCaseOnlyDir(t *testing.T) {
testPullCaseOnlyDirOrSymlink(t, true)
}
func TestPullCaseOnlySymlink(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("symlinks not supported on windows")
}
testPullCaseOnlyDirOrSymlink(t, false)
}
func testPullCaseOnlyDirOrSymlink(t *testing.T, dir bool) {
m, f := setupSendReceiveFolder()
defer cleanupSRFolder(f, m)
ffs := f.Filesystem()
name := "foo"
if dir {
must(t, ffs.Mkdir(name, 0777))
} else {
must(t, ffs.CreateSymlink("target", name))
}
must(t, f.scanSubdirs(nil))
var cur protocol.FileInfo
hasCur := false
snap := dbSnapshot(t, m, f.ID)
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if hasCur {
t.Fatal("got more than one file")
}
cur = i.(protocol.FileInfo)
hasCur = true
return true
})
if !hasCur {
t.Fatal("file is missing")
}
scanChan := make(chan string, 1)
dbUpdateChan := make(chan dbUpdateJob, 1)
remote := *(&cur)
remote.Version = protocol.Vector{}.Update(device1.Short())
remote.Name = strings.ToUpper(cur.Name)
if dir {
f.handleDir(remote, snap, dbUpdateChan, scanChan)
} else {
f.handleSymlink(remote, snap, dbUpdateChan, scanChan)
}
select {
case <-dbUpdateChan: // boring case sensitive filesystem
return
case <-scanChan:
t.Error("no need to scan anything here")
default:
}
if errStr, ok := f.tempPullErrors[remote.Name]; !ok {
t.Error("missing error for", remote.Name)
} else if !strings.Contains(errStr, "differs from name") {
t.Error("unexpected error", errStr, "for", remote.Name)
}
}
func cleanupSharedPullerState(s *sharedPullerState) {
s.mut.Lock()
defer s.mut.Unlock()

1
lib/model/folder_test.go
View File

@ -12,6 +12,7 @@ import (
"testing"
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/config"
)

115
lib/model/model_test.go
View File

@ -270,17 +270,15 @@ func BenchmarkRequestOut(b *testing.B) {
}
func BenchmarkRequestInSingleFile(b *testing.B) {
testOs := &fatalOs{b}
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
buf := make([]byte, 128<<10)
rand.Read(buf)
testOs.RemoveAll("testdata/request")
defer testOs.RemoveAll("testdata/request")
testOs.MkdirAll("testdata/request/for/a/file/in/a/couple/of/dirs", 0755)
ioutil.WriteFile("testdata/request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
mustRemove(b, defaultFs.RemoveAll("request"))
defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }()
must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755))
writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644)
b.ResetTimer()
@ -294,13 +292,11 @@ func BenchmarkRequestInSingleFile(b *testing.B) {
}
func TestDeviceRename(t *testing.T) {
testOs := &fatalOs{t}
hello := protocol.HelloResult{
ClientName: "syncthing",
ClientVersion: "v0.9.4",
}
defer testOs.Remove("testdata/tmpconfig.xml")
defer func() { mustRemove(t, defaultFs.Remove("tmpconfig.xml")) }()
rawCfg := config.New(device1)
rawCfg.Devices = []config.DeviceConfiguration{
@ -1447,12 +1443,10 @@ func changeIgnores(t *testing.T, m *model, expected []string) {
}
func TestIgnores(t *testing.T) {
testOs := &fatalOs{t}
// Assure a clean start state
testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644)
mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644)
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
@ -1504,18 +1498,16 @@ func TestIgnores(t *testing.T) {
// Make sure no .stignore file is considered valid
defer func() {
testOs.Rename("testdata/.stignore.bak", "testdata/.stignore")
must(t, defaultFs.Rename(".stignore.bak", ".stignore"))
}()
testOs.Rename("testdata/.stignore", "testdata/.stignore.bak")
must(t, defaultFs.Rename(".stignore", ".stignore.bak"))
changeIgnores(t, m, []string{})
}
func TestEmptyIgnores(t *testing.T) {
testOs := &fatalOs{t}
// Assure a clean start state
testOs.RemoveAll(filepath.Join("testdata", config.DefaultMarkerName))
testOs.MkdirAll(filepath.Join("testdata", config.DefaultMarkerName), 0644)
mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName))
must(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644))
m := setupModel(defaultCfgWrapper)
defer cleanupModel(m)
@ -2117,14 +2109,14 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
}
func TestIssue3028(t *testing.T) {
testOs := &fatalOs{t}
// Create two files that we'll delete, one with a name that is a prefix of the other.
must(t, ioutil.WriteFile("testdata/testrm", []byte("Hello"), 0644))
defer testOs.Remove("testdata/testrm")
must(t, ioutil.WriteFile("testdata/testrm2", []byte("Hello"), 0644))
defer testOs.Remove("testdata/testrm2")
must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644))
must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644))
defer func() {
mustRemove(t, defaultFs.Remove("testrm"))
mustRemove(t, defaultFs.Remove("testrm2"))
}()
// Create a model and default folder
@ -2138,8 +2130,8 @@ func TestIssue3028(t *testing.T) {
// Delete and rescan specifically these two
testOs.Remove("testdata/testrm")
testOs.Remove("testdata/testrm2")
must(t, defaultFs.Remove("testrm"))
must(t, defaultFs.Remove("testrm2"))
m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
// Verify that the number of files decreased by two and the number of
@ -2601,7 +2593,7 @@ func TestIssue2571(t *testing.T) {
must(t, testFs.RemoveAll("toLink"))
must(t, osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "linkTarget"), filepath.Join(testFs.URI(), "toLink")))
must(t, fs.DebugSymlinkForTestsOnly(testFs, testFs, "linkTarget", "toLink"))
m.ScanFolder("default")
@ -2718,13 +2710,10 @@ func TestCustomMarkerName(t *testing.T) {
{Name: "dummyfile"},
})
fcfg := config.FolderConfiguration{
ID: "default",
Path: "rwtestfolder",
Type: config.FolderTypeSendReceive,
RescanIntervalS: 1,
MarkerName: "myfile",
}
fcfg := testFolderConfigTmp()
fcfg.ID = "default"
fcfg.RescanIntervalS = 1
fcfg.MarkerName = "myfile"
cfg := createTmpWrapper(config.Configuration{
Folders: []config.FolderConfiguration{fcfg},
Devices: []config.DeviceConfiguration{
@ -2735,13 +2724,12 @@ func TestCustomMarkerName(t *testing.T) {
})
testOs.RemoveAll(fcfg.Path)
defer testOs.RemoveAll(fcfg.Path)
m := newModel(cfg, myID, "syncthing", "dev", ldb, nil)
sub := m.evLogger.Subscribe(events.StateChanged)
defer sub.Unsubscribe()
m.ServeBackground()
defer cleanupModel(m)
defer cleanupModelAndRemoveDir(m, fcfg.Path)
waitForState(t, sub, "default", "folder path missing")
@ -3806,6 +3794,57 @@ func TestBlockListMap(t *testing.T) {
}
}
func TestScanRenameCaseOnly(t *testing.T) {
wcfg, fcfg := tmpDefaultWrapper()
m := setupModel(wcfg)
defer cleanupModel(m)
ffs := fcfg.Filesystem()
name := "foo"
must(t, writeFile(ffs, name, []byte("contents"), 0644))
m.ScanFolders()
snap := dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found := false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if found {
t.Fatal("got more than one file")
}
if i.FileName() != name {
t.Fatalf("got file %v, expected %v", i.FileName(), name)
}
found = true
return true
})
snap.Release()
upper := strings.ToUpper(name)
must(t, ffs.Rename(name, upper))
m.ScanFolders()
snap = dbSnapshot(t, m, fcfg.ID)
defer snap.Release()
found = false
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
if i.FileName() == name {
if i.IsDeleted() {
return true
}
t.Fatal("renamed file not deleted")
}
if i.FileName() != upper {
t.Fatalf("got file %v, expected %v", i.FileName(), upper)
}
if found {
t.Fatal("got more than the expected files")
}
found = true
return true
})
}
func TestConnectionTerminationOnFolderAdd(t *testing.T) {
testConfigChangeClosesConnections(t, false, true, nil, func(cfg config.Wrapper) {
fcfg := testFolderConfigTmp()

12
lib/model/requests_test.go
View File

@ -323,7 +323,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
fc.deleteFile(invDel)
fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents)
fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents)
if err := ioutil.WriteFile(filepath.Join(fss.URI(), ignExisting), otherContents, 0644); err != nil {
if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil {
panic(err)
}
@ -465,12 +465,12 @@ func TestIssue4841(t *testing.T) {
func TestRescanIfHaveInvalidContent(t *testing.T) {
m, fc, fcfg := setupModelWithConnection()
tmpDir := fcfg.Filesystem().URI()
defer cleanupModelAndRemoveDir(m, tmpDir)
tfs := fcfg.Filesystem()
defer cleanupModelAndRemoveDir(m, tfs.URI())
payload := []byte("hello")
must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
must(t, writeFile(tfs, "foo", payload, 0777))
received := make(chan []protocol.FileInfo)
fc.mut.Lock()
@ -511,7 +511,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
payload = []byte("bye")
buf = make([]byte, len(payload))
must(t, ioutil.WriteFile(filepath.Join(tmpDir, "foo"), payload, 0777))
must(t, writeFile(tfs, "foo", payload, 0777))
_, err = m.Request(device1, "default", "foo", int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false)
if err == nil {
@ -1051,7 +1051,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) {
}
fc.mut.Unlock()
if err := ioutil.WriteFile(filepath.Join(fss.URI(), file), contents, 0644); err != nil {
if err := writeFile(fss, file, contents, 0644); err != nil {
panic(err)
}
m.ScanFolders()

9
lib/model/testos_test.go
View File

@ -9,6 +9,8 @@ package model
import (
"os"
"time"
"github.com/syncthing/syncthing/lib/fs"
)
// fatal is the required common interface between *testing.B and *testing.T
@ -28,6 +30,13 @@ func must(f fatal, err error) {
}
}
func mustRemove(f fatal, err error) {
f.Helper()
if err != nil && !fs.IsNotExist(err) {
f.Fatal(err)
}
}
func (f *fatalOs) Chmod(name string, mode os.FileMode) {
f.Helper()
must(f, os.Chmod(name, mode))

3
lib/model/testutils_test.go
View File

@ -36,9 +36,8 @@ func init() {
device1, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
device2, _ = protocol.DeviceIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
defaultFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
defaultFolderConfig = testFolderConfig("testdata")
defaultFs = defaultFolderConfig.Filesystem()
defaultCfgWrapper = createTmpWrapper(config.New(myID))
_, _ = defaultCfgWrapper.SetDevice(config.NewDeviceConfiguration(device1, "device1"))

6
lib/osutil/osutil.go
View File

@ -131,8 +131,10 @@ func copyFileContents(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, src
}
func IsDeleted(ffs fs.Filesystem, name string) bool {
if _, err := ffs.Lstat(name); fs.IsNotExist(err) {
return true
if _, err := ffs.Lstat(name); err != nil {
if fs.IsNotExist(err) || fs.IsErrCaseConflict(err) {
return true
}
}
switch TraversesSymlink(ffs, filepath.Dir(name)).(type) {
case *NotADirectoryError, *TraversesSymlinkError:

2
lib/osutil/osutil_test.go
View File

@ -62,7 +62,7 @@ func TestIsDeleted(t *testing.T) {
}
}
for _, n := range []string{"Dir", "File", "Del"} {
if err := osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), strings.ToLower(n)), filepath.Join(testFs.URI(), "linkTo"+n)); err != nil {
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, strings.ToLower(n), "linkTo"+n); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}

21
lib/osutil/symlink.go
View File

@ -1,21 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"os"
)
// DebugSymlinkForTestsOnly is not and should not be used in Syncthing code,
// hence the cumbersome name to make it obvious if this ever leaks. Its
// reason for existence is the Windows version, which allows creating
// symlinks when non-elevated.
func DebugSymlinkForTestsOnly(oldname, newname string) error {
return os.Symlink(oldname, newname)
}

14
lib/osutil/traversessymlink_test.go
View File

@ -24,9 +24,9 @@ func TestTraversesSymlink(t *testing.T) {
}
defer os.RemoveAll(tmpDir)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
fs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(fs.URI(), "a", "b"), filepath.Join(fs.URI(), "a", "l")); err != nil {
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}
@ -34,7 +34,7 @@ func TestTraversesSymlink(t *testing.T) {
}
// a/l -> b, so a/l/c should resolve by normal stat
info, err := fs.Lstat("a/l/c")
info, err := testFs.Lstat("a/l/c")
if err != nil {
t.Fatal("unexpected error", err)
}
@ -64,7 +64,7 @@ func TestTraversesSymlink(t *testing.T) {
}
for _, tc := range cases {
if res := osutil.TraversesSymlink(fs, tc.name); tc.traverses == (res == nil) {
if res := osutil.TraversesSymlink(testFs, tc.name); tc.traverses == (res == nil) {
t.Errorf("TraversesSymlink(%q) = %v, should be %v", tc.name, res, tc.traverses)
}
}
@ -78,8 +78,8 @@ func TestIssue4875(t *testing.T) {
defer os.RemoveAll(tmpDir)
testFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir)
testFs.MkdirAll("a/b/c", 0755)
if err = osutil.DebugSymlinkForTestsOnly(filepath.Join(testFs.URI(), "a", "b"), filepath.Join(testFs.URI(), "a", "l")); err != nil {
testFs.MkdirAll(filepath.Join("a", "b", "c"), 0755)
if err = fs.DebugSymlinkForTestsOnly(testFs, testFs, filepath.Join("a", "b"), filepath.Join("a", "l")); err != nil {
if runtime.GOOS == "windows" {
t.Skip("Symlinks aren't working")
}

4
lib/scanner/walk.go
View File

@ -540,6 +540,10 @@ func (w *walker) handleError(ctx context.Context, context, path string, err erro
}
}
func (w *walker) String() string {
return fmt.Sprintf("walker/%s@%p", w.Folder, w)
}
// A byteCounter gets bytes added to it via Update() and then provides the
// Total() and one minute moving average Rate() in bytes per second.
type byteCounter struct {

53
lib/scanner/walk_test.go
View File

@ -26,7 +26,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
@ -40,17 +39,19 @@ type testfile struct {
type testfileList []testfile
var testFs fs.Filesystem
var testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
var (
testFs fs.Filesystem
testFsType = fs.FilesystemTypeBasic
testdata = testfileList{
{"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
{"dir1", 128, ""},
{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
{"dir2", 128, ""},
{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
}
)
func init() {
// This test runs the risk of entering infinite recursion if it fails.
@ -270,7 +271,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
defer os.RemoveAll("_symlinks")
os.Symlink("../testdata", "_symlinks/link")
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
fs := fs.NewFilesystem(testFsType, "_symlinks")
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
@ -298,15 +299,15 @@ func TestWalkSymlinkWindows(t *testing.T) {
os.RemoveAll(name)
os.Mkdir(name, 0755)
defer os.RemoveAll(name)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name)
if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil {
testFs := fs.NewFilesystem(testFsType, name)
if err := fs.DebugSymlinkForTestsOnly(testFs, testFs, "../testdata", "link"); err != nil {
// Probably we require permissions we don't have.
t.Skip(err)
}
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil, 0)
files := walkDir(testFs, path, nil, nil, 0)
// Verify that we got zero symlinks
if len(files) != 0 {
@ -322,10 +323,12 @@ func TestWalkRootSymlink(t *testing.T) {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
testFs := fs.NewFilesystem(testFsType, tmp)
link := filepath.Join(tmp, "link")
link := "link"
dest, _ := filepath.Abs("testdata/dir1")
if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil {
destFs := fs.NewFilesystem(testFsType, dest)
if err := fs.DebugSymlinkForTestsOnly(destFs, testFs, ".", "link"); err != nil {
if runtime.GOOS == "windows" {
// Probably we require permissions we don't have.
t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
@ -335,15 +338,15 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan root with symlink at FS root
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
files := walkDir(fs.NewFilesystem(testFsType, filepath.Join(testFs.URI(), link)), ".", nil, nil, 0)
// Verify that we got two files
if len(files) != 2 {
t.Errorf("expected two files, not %d", len(files))
t.Fatalf("expected two files, not %d", len(files))
}
// Scan symlink below FS root
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), "link", nil, nil, 0)
files = walkDir(testFs, "link", nil, nil, 0)
// Verify that we got the one symlink, except on windows
if runtime.GOOS == "windows" {
@ -355,7 +358,7 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan path below symlink
files = walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
files = walkDir(fs.NewFilesystem(testFsType, tmp), filepath.Join("link", "cfile"), nil, nil, 0)
// Verify that we get nothing
if len(files) != 0 {
@ -554,7 +557,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), fs.NewFilesystem(testFsType, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@ -652,7 +655,7 @@ func TestIssue4799(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {
@ -714,7 +717,7 @@ func TestIssue4841(t *testing.T) {
}
defer os.RemoveAll(tmp)
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
fs := fs.NewFilesystem(testFsType, tmp)
fd, err := fs.Create("foo")
if err != nil {

1
lib/ur/contract/contract.go
View File

@ -128,6 +128,7 @@ type Report struct {
DisableFsync int `json:"disableFsync,omitempty" since:"3"`
BlockPullOrder map[string]int `json:"blockPullOrder,omitempty" since:"3"`
CopyRangeMethod map[string]int `json:"copyRangeMethod,omitempty" since:"3"`
CaseSensitiveFS int `json:"caseSensitiveFS,omitempty" since:"3"`
} `json:"folderUsesV3,omitempty" since:"3"`
GUIStats struct {

3
lib/ur/usage_report.go
View File

@ -269,6 +269,9 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
}
report.FolderUsesV3.BlockPullOrder[cfg.BlockPullOrder.String()]++
report.FolderUsesV3.CopyRangeMethod[cfg.CopyRangeMethod.String()]++
if cfg.CaseSensitiveFS {
report.FolderUsesV3.CaseSensitiveFS++
}
}
sort.Ints(report.FolderUsesV3.FsWatcherDelays)

101
test/transfer-bench_test.go
View File

@ -10,42 +10,42 @@ package integration
import (
"log"
"math/rand"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/rc"
)
func TestBenchmarkTransferManyFiles(t *testing.T) {
benchmarkTransfer(t, 10000, 15)
setupAndBenchmarkTransfer(t, 10000, 15)
}
func TestBenchmarkTransferLargeFile1G(t *testing.T) {
benchmarkTransfer(t, 1, 30)
setupAndBenchmarkTransfer(t, 1, 30)
}
func TestBenchmarkTransferLargeFile2G(t *testing.T) {
benchmarkTransfer(t, 1, 31)
setupAndBenchmarkTransfer(t, 1, 31)
}
func TestBenchmarkTransferLargeFile4G(t *testing.T) {
benchmarkTransfer(t, 1, 32)
setupAndBenchmarkTransfer(t, 1, 32)
}
func TestBenchmarkTransferLargeFile8G(t *testing.T) {
benchmarkTransfer(t, 1, 33)
setupAndBenchmarkTransfer(t, 1, 33)
}
func TestBenchmarkTransferLargeFile16G(t *testing.T) {
benchmarkTransfer(t, 1, 34)
setupAndBenchmarkTransfer(t, 1, 34)
}
func TestBenchmarkTransferLargeFile32G(t *testing.T) {
benchmarkTransfer(t, 1, 35)
setupAndBenchmarkTransfer(t, 1, 35)
}
func benchmarkTransfer(t *testing.T, files, sizeExp int) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
func setupAndBenchmarkTransfer(t *testing.T, files, sizeExp int) {
cleanBenchmarkTransfer(t)
log.Println("Generating files...")
var err error
if files == 1 {
// Special case. Generate one file with the specified size exactly.
var fd *os.File
@ -57,13 +57,39 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
if err != nil {
t.Fatal(err)
}
err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp))
err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp), time.Now())
} else {
err = generateFiles("s1", files, sizeExp, "../LICENSE")
}
if err != nil {
t.Fatal(err)
}
benchmarkTransfer(t)
}
// TestBenchmarkTransferSameFiles doesn't actually transfer anything, but tests
// how fast two devicees get in sync if they have the same data locally.
func TestBenchmarkTransferSameFiles(t *testing.T) {
cleanBenchmarkTransfer(t)
t0 := time.Now()
rand.Seed(0)
log.Println("Generating files in s1...")
if err := generateFilesWithTime("s1", 10000, 10, "../LICENSE", t0); err != nil {
t.Fatal(err)
}
rand.Seed(0)
log.Println("Generating same files in s2...")
if err := generateFilesWithTime("s2", 10000, 10, "../LICENSE", t0); err != nil {
t.Fatal(err)
}
benchmarkTransfer(t)
}
func benchmarkTransfer(t *testing.T) {
expected, err := directoryContents("s1")
if err != nil {
t.Fatal(err)
@ -86,9 +112,9 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
sender.ResumeAll()
receiver.ResumeAll()
var t0, t1 time.Time
t0 := time.Now()
var t1 time.Time
lastEvent := 0
oneItemFinished := false
loop:
for {
@ -105,35 +131,22 @@ loop:
switch ev.Type {
case "ItemFinished":
oneItemFinished = true
continue
case "StateChanged":
data := ev.Data.(map[string]interface{})
if data["folder"].(string) != "default" {
continue
}
switch data["to"].(string) {
case "syncing":
t0 = ev.Time
continue
case "idle":
if !oneItemFinished {
continue
}
if !t0.IsZero() {
t1 = ev.Time
break loop
}
}
break loop
}
}
time.Sleep(250 * time.Millisecond)
}
processes := []*rc.Process{sender, receiver}
for {
if rc.InSync("default", processes...) {
t1 = time.Now()
break
}
time.Sleep(250 * time.Millisecond)
}
sendProc, err := sender.Stop()
if err != nil {
t.Fatal(err)
@ -159,4 +172,14 @@ loop:
printUsage("Receiver", recvProc, total)
printUsage("Sender", sendProc, total)
cleanBenchmarkTransfer(t)
}
func cleanBenchmarkTransfer(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
}

10
test/util.go
View File

@ -41,6 +41,10 @@ const (
)
func generateFiles(dir string, files, maxexp int, srcname string) error {
return generateFilesWithTime(dir, files, maxexp, srcname, time.Now())
}
func generateFilesWithTime(dir string, files, maxexp int, srcname string, t0 time.Time) error {
fd, err := os.Open(srcname)
if err != nil {
return err
@ -69,7 +73,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
}
s += rand.Int63n(a)
if err := generateOneFile(fd, p1, s); err != nil {
if err := generateOneFile(fd, p1, s, t0); err != nil {
return err
}
}
@ -77,7 +81,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
return nil
}
func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
func generateOneFile(fd io.ReadSeeker, p1 string, s int64, t0 time.Time) error {
src := io.LimitReader(&inifiteReader{fd}, int64(s))
dst, err := os.Create(p1)
if err != nil {
@ -96,7 +100,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
t := t0.Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
err = os.Chtimes(p1, t, t)
if err != nil {
return err