Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/versioner: Restore for all versioners, cross-device support (#5514) 

* lib/versioner: Restore for all versioners, cross-device support

Fixes #4631
Fixes #4586
Fixes #1634
Fixes #5338
Fixes #5419
This commit is contained in:
Audrius Butkevicius 2019-04-28 23:30:16 +01:00 committed by GitHub
parent 2984d40641
commit 0ca1f26ff8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 636 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
lib/fs/util.go
View File

@ -103,3 +103,55 @@ func IsParent(path, parent string) bool {
}
return strings.HasPrefix(path, parent)
}
func CommonPrefix(first, second string) string {
if filepath.IsAbs(first) != filepath.IsAbs(second) {
// Whatever
return ""
}
firstParts := strings.Split(filepath.Clean(first), string(PathSeparator))
secondParts := strings.Split(filepath.Clean(second), string(PathSeparator))
isAbs := filepath.IsAbs(first) && filepath.IsAbs(second)
count := len(firstParts)
if len(secondParts) < len(firstParts) {
count = len(secondParts)
}
common := make([]string, 0, count)
for i := 0; i < count; i++ {
if firstParts[i] != secondParts[i] {
break
}
common = append(common, firstParts[i])
}
if isAbs {
if runtime.GOOS == "windows" && isVolumeNameOnly(common) {
// Because strings.Split strips out path separators, if we're at the volume name, we end up without a separator
// Wedge an empty element to be joined with.
common = append(common, "")
} else if len(common) == 1 {
// If isAbs on non Windows, first element in both first and second is "", hence joining that returns nothing.
return string(PathSeparator)
}
}
// This should only be true on Windows when drive letters are different or when paths are relative.
// In case of UNC paths we should end up with more than a single element hence joining is fine
if len(common) == 0 {
return ""
}
// This has to be strings.Join, because filepath.Join([]string{"", "", "?", "C:", "Audrius"}...) returns garbage
result := strings.Join(common, string(PathSeparator))
return filepath.Clean(result)
}
func isVolumeNameOnly(parts []string) bool {
isNormalVolumeName := len(parts) == 1 && strings.HasSuffix(parts[0], ":")
isUNCVolumeName := len(parts) == 4 && strings.HasSuffix(parts[3], ":")
return isNormalVolumeName || isUNCVolumeName
}

46
lib/fs/util_test.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package fs
import (
"runtime"
"testing"
)
func TestCommonPrefix(t *testing.T) {
test := func(first, second, expect string) {
t.Helper()
res := CommonPrefix(first, second)
if res != expect {
t.Errorf("Expected %s got %s", expect, res)
}
}
if runtime.GOOS == "windows" {
test(`c:\Audrius\Downloads`, `c:\Audrius\Docs`, `c:\Audrius`)
test(`c:\Audrius\Downloads`, `C:\Audrius\Docs`, ``) // Case differences :(
test(`C:\Audrius-a\Downloads`, `C:\Audrius-b\Docs`, `C:\`)
test(`\\?\C:\Audrius-a\Downloads`, `\\?\C:\Audrius-b\Docs`, `\\?\C:\`)
test(`\\?\C:\Audrius\Downloads`, `\\?\C:\Audrius\Docs`, `\\?\C:\Audrius`)
test(`Audrius-a\Downloads`, `Audrius-b\Docs`, ``)
test(`Audrius\Downloads`, `Audrius\Docs`, `Audrius`)
test(`c:\Audrius\Downloads`, `Audrius\Docs`, ``)
test(`c:\`, `c:\`, `c:\`)
test(`\\?\c:\`, `\\?\c:\`, `\\?\c:\`)
} else {
test(`/Audrius/Downloads`, `/Audrius/Docs`, `/Audrius`)
test(`/Audrius\Downloads`, `/Audrius\Docs`, `/`)
test(`/Audrius-a/Downloads`, `/Audrius-b/Docs`, `/`)
test(`Audrius\Downloads`, `Audrius\Docs`, ``) // Windows separators
test(`Audrius/Downloads`, `Audrius/Docs`, `Audrius`)
test(`Audrius-a\Downloads`, `Audrius-b\Docs`, ``)
test(`/Audrius/Downloads`, `Audrius/Docs`, ``)
test(`/`, `/`, `/`)
}
test(`Audrius`, `Audrius`, `Audrius`)
test(`.`, `.`, `.`)
}

6
lib/model/folder_sendrecv.go
View File

@ -941,13 +941,13 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
if f.versioner != nil {
err = f.CheckAvailableSpace(source.Size)
if err == nil {
err = osutil.Copy(f.fs, source.Name, tempName)
err = osutil.Copy(f.fs, f.fs, source.Name, tempName)
if err == nil {
err = osutil.InWritableDir(f.versioner.Archive, f.fs, source.Name)
}
}
} else {
err = osutil.TryRename(f.fs, source.Name, tempName)
err = osutil.RenameOrCopy(f.fs, f.fs, source.Name, tempName)
}
if err != nil {
return err
@ -1510,7 +1510,7 @@ func (f *sendReceiveFolder) performFinish(file, curFile protocol.FileInfo, hasCu
// Replace the original content with the new one. If it didn't work,
// leave the temp file in place for reuse.
if err := osutil.TryRename(f.fs, tempName, file.Name); err != nil {
if err := osutil.RenameOrCopy(f.fs, f.fs, tempName, file.Name); err != nil {
return err
}

111
lib/model/model.go
View File

@ -2310,58 +2310,12 @@ func (m *model) GetFolderVersions(folder string) (map[string][]versioner.FileVer
return nil, errFolderMissing
}
files := make(map[string][]versioner.FileVersion)
filesystem := fcfg.Filesystem()
err := filesystem.Walk(".stversions", func(path string, f fs.FileInfo, err error) error {
// Skip root (which is ok to be a symlink)
if path == ".stversions" {
return nil
}
// Skip walking if we cannot walk...
if err != nil {
return err
}
// Ignore symlinks
if f.IsSymlink() {
return fs.SkipDir
}
// No records for directories
if f.IsDir() {
return nil
}
// Strip .stversions prefix.
path = strings.TrimPrefix(path, ".stversions"+string(fs.PathSeparator))
name, tag := versioner.UntagFilename(path)
// Something invalid
if name == "" || tag == "" {
return nil
}
name = osutil.NormalizedFilename(name)
versionTime, err := time.ParseInLocation(versioner.TimeFormat, tag, locationLocal)
if err != nil {
return nil
}
files[name] = append(files[name], versioner.FileVersion{
VersionTime: versionTime.Truncate(time.Second),
ModTime: f.ModTime().Truncate(time.Second),
Size: f.Size(),
})
return nil
})
if err != nil {
return nil, err
ver := fcfg.Versioner()
if ver == nil {
return nil, errors.New("no versioner configured")
}
return files, nil
return ver.GetVersions()
}
func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
@ -2370,69 +2324,22 @@ func (m *model) RestoreFolderVersions(folder string, versions map[string]time.Ti
return nil, errFolderMissing
}
filesystem := fcfg.Filesystem()
ver := fcfg.Versioner()
restore := make(map[string]string)
errors := make(map[string]string)
restoreErrors := make(map[string]string)
// Validation
for file, version := range versions {
file = osutil.NativeFilename(file)
tag := version.In(locationLocal).Truncate(time.Second).Format(versioner.TimeFormat)
versionedTaggedFilename := filepath.Join(".stversions", versioner.TagFilename(file, tag))
// Check that the thing we've been asked to restore is actually a file
// and that it exists.
if info, err := filesystem.Lstat(versionedTaggedFilename); err != nil {
errors[file] = err.Error()
continue
} else if !info.IsRegular() {
errors[file] = "not a file"
continue
}
// Check that the target location of where we are supposed to restore
// either does not exist, or is actually a file.
if info, err := filesystem.Lstat(file); err == nil && !info.IsRegular() {
errors[file] = "cannot replace a non-file"
continue
} else if err != nil && !fs.IsNotExist(err) {
errors[file] = err.Error()
continue
}
restore[file] = versionedTaggedFilename
}
// Execution
var err error
for target, source := range restore {
err = nil
if _, serr := filesystem.Lstat(target); serr == nil {
if ver != nil {
err = osutil.InWritableDir(ver.Archive, filesystem, target)
} else {
err = osutil.InWritableDir(filesystem.Remove, filesystem, target)
}
}
filesystem.MkdirAll(filepath.Dir(target), 0755)
if err == nil {
err = osutil.Copy(filesystem, source, target)
}
if err != nil {
errors[target] = err.Error()
continue
if err := ver.Restore(file, version); err != nil {
restoreErrors[file] = err.Error()
}
}
// Trigger scan
if !fcfg.FSWatcherEnabled {
m.ScanFolder(folder)
go func() { _ = m.ScanFolder(folder) }()
}
return errors, nil
return restoreErrors, nil
}
func (m *model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {

9
lib/model/model_test.go
View File

@ -3151,8 +3151,8 @@ func TestVersionRestore(t *testing.T) {
".stversions/dir/file~20171210-040406.txt",
".stversions/very/very/deep/one~20171210-040406.txt", // lives deep down, no directory exists.
".stversions/dir/existing~20171210-040406.txt", // exists, should expect to be archived.
".stversions/dir/file.txt~20171210-040405", // incorrect tag format, ignored.
".stversions/dir/cat", // incorrect tag format, ignored.
".stversions/dir/file.txt~20171210-040405", // old tag format, supported
".stversions/dir/cat", // untagged which was used by trashcan, supported
// "file.txt" will be restored
"existing",
@ -3182,9 +3182,10 @@ func TestVersionRestore(t *testing.T) {
"file.txt": 1,
"existing": 1,
"something": 1,
"dir/file.txt": 3,
"dir/file.txt": 4,
"dir/existing.txt": 1,
"very/very/deep/one.txt": 1,
"dir/cat": 1,
}
for name, vers := range versions {
@ -3229,7 +3230,7 @@ func TestVersionRestore(t *testing.T) {
ferr, err := m.RestoreFolderVersions("default", restore)
must(t, err)
if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot replace a non-file" {
if err, ok := ferr["something"]; len(ferr) > 1 || !ok || err != "cannot restore on top of a directory" {
t.Fatalf("incorrect error or count: %d %s", len(ferr), ferr)
}

71
lib/osutil/osutil.go
View File

@ -22,37 +22,62 @@ import (
// often enough that there is any contention on this lock.
var renameLock = sync.NewMutex()
// TryRename renames a file, leaving source file intact in case of failure.
// RenameOrCopy renames a file, leaving source file intact in case of failure.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func TryRename(filesystem fs.Filesystem, from, to string) error {
func RenameOrCopy(src, dst fs.Filesystem, from, to string) error {
renameLock.Lock()
defer renameLock.Unlock()
return withPreparedTarget(filesystem, from, to, func() error {
return filesystem.Rename(from, to)
})
}
return withPreparedTarget(dst, from, to, func() error {
// Optimisation 1
if src.Type() == dst.Type() && src.URI() == dst.URI() {
return src.Rename(from, to)
}
// Rename moves a temporary file to its final place.
// Will make sure to delete the from file if the operation fails, so use only
// for situations like committing a temp file to its final location.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Rename(filesystem fs.Filesystem, from, to string) error {
// Don't leave a dangling temp file in case of rename error
if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) {
defer filesystem.Remove(from)
}
return TryRename(filesystem, from, to)
// "Optimisation" 2
// Try to find a common prefix between the two filesystems, use that as the base for the new one
// and try a rename.
if src.Type() == dst.Type() {
commonPrefix := fs.CommonPrefix(src.URI(), dst.URI())
if len(commonPrefix) > 0 {
commonFs := fs.NewFilesystem(src.Type(), commonPrefix)
err := commonFs.Rename(
filepath.Join(strings.TrimPrefix(src.URI(), commonPrefix), from),
filepath.Join(strings.TrimPrefix(dst.URI(), commonPrefix), to),
)
if err == nil {
return nil
}
}
}
// Everything is sad, do a copy and delete.
if _, err := dst.Stat(to); !fs.IsNotExist(err) {
err := dst.Remove(to)
if err != nil {
return err
}
}
err := copyFileContents(src, dst, from, to)
if err != nil {
_ = dst.Remove(to)
return err
}
return withPreparedTarget(src, from, from, func() error {
return src.Remove(from)
})
})
}
// Copy copies the file content from source to destination.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Copy(filesystem fs.Filesystem, from, to string) (err error) {
return withPreparedTarget(filesystem, from, to, func() error {
return copyFileContents(filesystem, from, to)
func Copy(src, dst fs.Filesystem, from, to string) (err error) {
return withPreparedTarget(dst, from, to, func() error {
return copyFileContents(src, dst, from, to)
})
}
@ -115,13 +140,13 @@ func withPreparedTarget(filesystem fs.Filesystem, from, to string, f func() erro
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all its contents will be replaced by the contents
// of the source file.
func copyFileContents(filesystem fs.Filesystem, src, dst string) (err error) {
in, err := filesystem.Open(src)
func copyFileContents(srcFs, dstFs fs.Filesystem, src, dst string) (err error) {
in, err := srcFs.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := filesystem.Create(dst)
out, err := dstFs.Create(dst)
if err != nil {
return
}

79
lib/osutil/osutil_test.go
View File

@ -7,6 +7,7 @@
package osutil_test
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
@ -192,7 +193,7 @@ func TestInWritableDirWindowsRename(t *testing.T) {
}
rename := func(path string) error {
return osutil.Rename(fs, path, path+"new")
return osutil.RenameOrCopy(fs, fs, path, path+"new")
}
for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
@ -268,3 +269,79 @@ func TestIsDeleted(t *testing.T) {
testFs.Chmod("inacc", 0777)
os.RemoveAll("testdata")
}
func TestRenameOrCopy(t *testing.T) {
mustTempDir := func() string {
t.Helper()
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
return tmpDir
}
sameFs := fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir())
tests := []struct {
src fs.Filesystem
dst fs.Filesystem
file string
}{
{
src: sameFs,
dst: sameFs,
file: "file",
},
{
src: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
dst: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
file: "file",
},
{
src: fs.NewFilesystem(fs.FilesystemTypeFake, `fake://fake/?files=1&seed=42`),
dst: fs.NewFilesystem(fs.FilesystemTypeBasic, mustTempDir()),
file: osutil.NativeFilename(`05/7a/4d52f284145b9fe8`),
},
}
for _, test := range tests {
content := test.src.URI()
if _, err := test.src.Lstat(test.file); err != nil {
if !fs.IsNotExist(err) {
t.Fatal(err)
}
if fd, err := test.src.Create(test.file); err != nil {
t.Fatal(err)
} else {
if _, err := fd.Write([]byte(test.src.URI())); err != nil {
t.Fatal(err)
}
_ = fd.Close()
}
} else {
fd, err := test.src.Open(test.file)
if err != nil {
t.Fatal(err)
}
buf, err := ioutil.ReadAll(fd)
if err != nil {
t.Fatal(err)
}
_ = fd.Close()
content = string(buf)
}
err := osutil.RenameOrCopy(test.src, test.dst, test.file, "new")
if err != nil {
t.Fatal(err)
}
if fd, err := test.dst.Open("new"); err != nil {
t.Fatal(err)
} else {
if buf, err := ioutil.ReadAll(fd); err != nil {
t.Fatal(err)
} else if string(buf) != content {
t.Fatalf("expected %s got %s", content, string(buf))
}
}
}
}

9
lib/versioner/external.go
View File

@ -12,6 +12,7 @@ import (
"os/exec"
"runtime"
"strings"
"time"
"github.com/syncthing/syncthing/lib/fs"
@ -103,3 +104,11 @@ func (v External) Archive(filePath string) error {
}
return errors.New("Versioner: file was not removed by external script")
}
func (v External) GetVersions() (map[string][]FileVersion, error) {
return nil, ErrRestorationNotSupported
}
func (v External) Restore(filePath string, versionTime time.Time) error {
return ErrRestorationNotSupported
}

68
lib/versioner/simple.go
View File

@ -9,9 +9,9 @@ package versioner
import (
"path/filepath"
"strconv"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
)
@ -21,19 +21,21 @@ func init() {
}
type Simple struct {
keep int
fs fs.Filesystem
keep int
folderFs fs.Filesystem
versionsFs fs.Filesystem
}
func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
func NewSimple(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
keep, err := strconv.Atoi(params["keep"])
if err != nil {
keep = 5 // A reasonable default
}
s := Simple{
keep: keep,
fs: fs,
keep: keep,
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
}
l.Debugf("instantiated %#v", s)
@ -43,51 +45,17 @@ func NewSimple(folderID string, fs fs.Filesystem, params map[string]string) Vers
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (v Simple) Archive(filePath string) error {
info, err := v.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename)
if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := ".stversions"
_, err = v.fs.Stat(versionsDir)
if err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir .stversions")
v.fs.Mkdir(versionsDir, 0755)
v.fs.Hide(versionsDir)
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
dir := filepath.Join(versionsDir, inFolderPath)
err = v.fs.MkdirAll(dir, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := TagFilename(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(dir, ver)
l.Debugln("moving to", dst)
err = osutil.Rename(v.fs, filePath, dst)
if err != nil {
return err
}
dir := filepath.Dir(filePath)
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(dir, TagFilename(file, TimeGlob))
newVersions, err := v.fs.Glob(pattern)
newVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@ -95,7 +63,7 @@ func (v Simple) Archive(filePath string) error {
// Also according to the old file.ext~timestamp pattern.
pattern = filepath.Join(dir, file+"~"+TimeGlob)
oldVersions, err := v.fs.Glob(pattern)
oldVersions, err := v.versionsFs.Glob(pattern)
if err != nil {
l.Warnln("globbing:", err, "for", pattern)
return nil
@ -108,7 +76,7 @@ func (v Simple) Archive(filePath string) error {
if len(versions) > v.keep {
for _, toRemove := range versions[:len(versions)-v.keep] {
l.Debugln("cleaning out", toRemove)
err = v.fs.Remove(toRemove)
err = v.versionsFs.Remove(toRemove)
if err != nil {
l.Warnln("removing old version:", err)
}
@ -117,3 +85,11 @@ func (v Simple) Archive(filePath string) error {
return nil
}
func (v Simple) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(v.versionsFs)
}
func (v Simple) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}

65
lib/versioner/staggered.go
View File

@ -7,7 +7,6 @@
package versioner
import (
"os"
"path/filepath"
"strconv"
"time"
@ -48,16 +47,9 @@ func NewStaggered(folderID string, folderFs fs.Filesystem, params map[string]str
cleanInterval = 3600 // Default: clean once per hour
}
// Use custom path if set, otherwise .stversions in folderPath
var versionsFs fs.Filesystem
if params["versionsPath"] == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if filepath.IsAbs(params["versionsPath"]) {
versionsFs = fs.NewFilesystem(folderFs.Type(), params["versionsPath"])
} else {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), params["versionsPath"]))
}
l.Debugln("%s folder using %s (%s) staggered versioner dir", folderID, versionsFs.URI(), versionsFs.Type())
// Backwards compatibility
params["fsPath"] = params["versionsPath"]
versionsFs := fsFromParams(folderFs, params)
s := &Staggered{
cleanInterval: cleanInterval,
@ -225,53 +217,12 @@ func (v *Staggered) Archive(filePath string) error {
v.mutex.Lock()
defer v.mutex.Unlock()
info, err := v.folderFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
if err := archiveFile(v.folderFs, v.versionsFs, filePath, TagFilename); err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
if _, err := v.versionsFs.Stat("."); err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir", v.versionsFs)
v.versionsFs.MkdirAll(".", 0755)
v.versionsFs.Hide(".")
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
if err != nil {
return err
}
err = v.versionsFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := TagFilename(file, time.Now().Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
/// TODO: Fix this when we have an alternative filesystem implementation
if v.versionsFs.Type() != fs.FilesystemTypeBasic {
panic("bug: staggered versioner used with unsupported filesystem")
}
err = os.Rename(filepath.Join(v.folderFs.URI(), filePath), filepath.Join(v.versionsFs.URI(), dst))
if err != nil {
return err
}
// Glob according to the new file~timestamp.ext pattern.
pattern := filepath.Join(inFolderPath, TagFilename(file, TimeGlob))
@ -295,3 +246,11 @@ func (v *Staggered) Archive(filePath string) error {
return nil
}
func (v *Staggered) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(v.versionsFs)
}
func (v *Staggered) Restore(filepath string, versionTime time.Time) error {
return restoreFile(v.versionsFs, v.folderFs, filepath, versionTime, TagFilename)
}

96
lib/versioner/trashcan.go
View File

@ -8,12 +8,10 @@ package versioner
import (
"fmt"
"path/filepath"
"strconv"
"time"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func init() {
@ -22,17 +20,19 @@ func init() {
}
type Trashcan struct {
fs fs.Filesystem
folderFs fs.Filesystem
versionsFs fs.Filesystem
cleanoutDays int
stop chan struct{}
}
func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Versioner {
func NewTrashcan(folderID string, folderFs fs.Filesystem, params map[string]string) Versioner {
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
// On error we default to 0, "do not clean out the trash can"
s := &Trashcan{
fs: fs,
folderFs: folderFs,
versionsFs: fsFromParams(folderFs, params),
cleanoutDays: cleanoutDays,
stop: make(chan struct{}),
}
@ -44,49 +44,9 @@ func NewTrashcan(folderID string, fs fs.Filesystem, params map[string]string) Ve
// Archive moves the named file away to a version archive. If this function
// returns nil, the named file does not exist any more (has been archived).
func (t *Trashcan) Archive(filePath string) error {
info, err := t.fs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
versionsDir := ".stversions"
if _, err := t.fs.Stat(versionsDir); err != nil {
if !fs.IsNotExist(err) {
return err
}
l.Debugln("creating versions dir", versionsDir)
if err := t.fs.MkdirAll(versionsDir, 0777); err != nil {
return err
}
t.fs.Hide(versionsDir)
}
l.Debugln("archiving", filePath)
archivedPath := filepath.Join(versionsDir, filePath)
if err := t.fs.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !fs.IsExist(err) {
return err
}
l.Debugln("moving to", archivedPath)
if err := osutil.Rename(t.fs, filePath, archivedPath); err != nil {
return err
}
// Set the mtime to the time the file was deleted. This is used by the
// cleanout routine. If this fails things won't work optimally but there's
// not much we can do about it so we ignore the error.
t.fs.Chtimes(archivedPath, time.Now(), time.Now())
return nil
return archiveFile(t.folderFs, t.versionsFs, filePath, func(name, tag string) string {
return name
})
}
func (t *Trashcan) Serve() {
@ -124,8 +84,7 @@ func (t *Trashcan) String() string {
}
func (t *Trashcan) cleanoutArchive() error {
versionsDir := ".stversions"
if _, err := t.fs.Lstat(versionsDir); fs.IsNotExist(err) {
if _, err := t.versionsFs.Lstat("."); fs.IsNotExist(err) {
return nil
}
@ -144,20 +103,45 @@ func (t *Trashcan) cleanoutArchive() error {
if info.ModTime().Before(cutoff) {
// The file is too old; remove it.
t.fs.Remove(path)
err = t.versionsFs.Remove(path)
} else {
// Keep this file, and remember it so we don't unnecessarily try
// to remove this directory.
dirTracker.addFile(path)
}
return nil
}
if err := t.fs.Walk(versionsDir, walkFn); err != nil {
return err
}
dirTracker.deleteEmptyDirs(t.fs)
if err := t.versionsFs.Walk(".", walkFn); err != nil {
return err
}
dirTracker.deleteEmptyDirs(t.versionsFs)
return nil
}
func (t *Trashcan) GetVersions() (map[string][]FileVersion, error) {
return retrieveVersions(t.versionsFs)
}
func (t *Trashcan) Restore(filepath string, versionTime time.Time) error {
// If we have an untagged file A and want to restore it on top of existing file A, we can't first archive the
// existing A as we'd overwrite the old A version, therefore when we archive existing file, we archive it with a
// tag but when the restoration is finished, we rename it (untag it). This is only important if when restoring A,
// there already exists a file at the same location
taggedName := ""
tagger := func(name, tag string) string {
// We can't use TagFilename here, as restoreFii would discover that as a valid version and restore that instead.
taggedName = fs.TempName(name)
return taggedName
}
err := restoreFile(t.versionsFs, t.folderFs, filepath, versionTime, tagger)
if taggedName == "" {
return err
}
return t.versionsFs.Rename(taggedName, filepath)
}

84
lib/versioner/trashcan_test.go
View File

@ -75,3 +75,87 @@ func TestTrashcanCleanout(t *testing.T) {
t.Error("empty directory should have been removed")
}
}
func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) {
// This tests that trashcan versioner restoration correctly archives existing file, because trashcan versioner
// files are untagged, archiving existing file to replace with a restored version technically should collide in
// in names.
tmpDir1, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
tmpDir2, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
folderFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir1)
versionsFs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmpDir2)
writeFile(t, folderFs, "file", "A")
versioner := NewTrashcan("", folderFs, map[string]string{
"fsType": "basic",
"fsPath": tmpDir2,
})
if err := versioner.Archive("file"); err != nil {
t.Fatal(err)
}
if _, err := folderFs.Stat("file"); !fs.IsNotExist(err) {
t.Fatal(err)
}
versionInfo, err := versionsFs.Stat("file")
if err != nil {
t.Fatal(err)
}
if content := readFile(t, versionsFs, "file"); content != "A" {
t.Errorf("expected A got %s", content)
}
writeFile(t, folderFs, "file", "B")
if err := versioner.Restore("file", versionInfo.ModTime().Truncate(time.Second)); err != nil {
t.Fatal(err)
}
if content := readFile(t, folderFs, "file"); content != "A" {
t.Errorf("expected A got %s", content)
}
if content := readFile(t, versionsFs, "file"); content != "B" {
t.Errorf("expected B got %s", content)
}
}
func readFile(t *testing.T, filesystem fs.Filesystem, name string) string {
fd, err := filesystem.Open(name)
if err != nil {
t.Fatal(err)
}
defer fd.Close()
buf, err := ioutil.ReadAll(fd)
if err != nil {
t.Fatal(err)
}
return string(buf)
}
func writeFile(t *testing.T, filesystem fs.Filesystem, name, content string) {
fd, err := filesystem.OpenFile(name, fs.OptReadWrite|fs.OptCreate, 0777)
if err != nil {
t.Fatal(err)
}
defer fd.Close()
if err := fd.Truncate(int64(len(content))); err != nil {
t.Fatal(err)
}
if n, err := fd.Write([]byte(content)); err != nil || n != len(content) {
t.Fatal(n, len(content), err)
}
}

225
lib/versioner/util.go
View File

@ -7,11 +7,30 @@
package versioner
import (
"fmt"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
var locationLocal *time.Location
var errDirectory = fmt.Errorf("cannot restore on top of a directory")
var errNotFound = fmt.Errorf("version not found")
var errFileAlreadyExists = fmt.Errorf("file already exists")
func init() {
var err error
locationLocal, err = time.LoadLocation("Local")
if err != nil {
panic(err.Error())
}
}
// Inserts ~tag just before the extension of the filename.
func TagFilename(name, tag string) string {
dir, file := filepath.Dir(name), filepath.Base(name)
@ -38,11 +57,215 @@ func UntagFilename(path string) (string, string) {
versionTag := ExtractTag(path)
// Files tagged with old style tags cannot be untagged.
if versionTag == "" || strings.HasSuffix(ext, versionTag) {
if versionTag == "" {
return "", ""
}
// Old style tag
if strings.HasSuffix(ext, versionTag) {
return strings.TrimSuffix(path, "~"+versionTag), versionTag
}
withoutExt := path[:len(path)-len(ext)-len(versionTag)-1]
name := withoutExt + ext
return name, versionTag
}
func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error) {
files := make(map[string][]FileVersion)
err := fileSystem.Walk(".", func(path string, f fs.FileInfo, err error) error {
// Skip root (which is ok to be a symlink)
if path == "." {
return nil
}
// Skip walking if we cannot walk...
if err != nil {
return err
}
// Ignore symlinks
if f.IsSymlink() {
return fs.SkipDir
}
// No records for directories
if f.IsDir() {
return nil
}
path = osutil.NormalizedFilename(path)
name, tag := UntagFilename(path)
// Something invalid, assume it's an untagged file
if name == "" || tag == "" {
versionTime := f.ModTime().Truncate(time.Second)
files[path] = append(files[path], FileVersion{
VersionTime: versionTime,
ModTime: versionTime,
Size: f.Size(),
})
return nil
}
versionTime, err := time.ParseInLocation(TimeFormat, tag, locationLocal)
if err != nil {
// Can't parse it, welp, continue
return nil
}
if err == nil {
files[name] = append(files[name], FileVersion{
VersionTime: versionTime.Truncate(time.Second),
ModTime: f.ModTime().Truncate(time.Second),
Size: f.Size(),
})
}
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
type fileTagger func(string, string) string
func archiveFile(srcFs, dstFs fs.Filesystem, filePath string, tagger fileTagger) error {
filePath = osutil.NativeFilename(filePath)
info, err := srcFs.Lstat(filePath)
if fs.IsNotExist(err) {
l.Debugln("not archiving nonexistent file", filePath)
return nil
} else if err != nil {
return err
}
if info.IsSymlink() {
panic("bug: attempting to version a symlink")
}
_, err = dstFs.Stat(".")
if err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir")
err := dstFs.Mkdir(".", 0755)
if err != nil {
return err
}
_ = dstFs.Hide(".")
} else {
return err
}
}
l.Debugln("archiving", filePath)
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
err = dstFs.MkdirAll(inFolderPath, 0755)
if err != nil && !fs.IsExist(err) {
return err
}
ver := tagger(file, info.ModTime().Format(TimeFormat))
dst := filepath.Join(inFolderPath, ver)
l.Debugln("moving to", dst)
err = osutil.RenameOrCopy(srcFs, dstFs, filePath, dst)
// Set the mtime to the time the file was deleted. This can be used by the
// cleanout routine. If this fails things won't work optimally but there's
// not much we can do about it so we ignore the error.
_ = dstFs.Chtimes(dst, time.Now(), time.Now())
return err
}
func restoreFile(src, dst fs.Filesystem, filePath string, versionTime time.Time, tagger fileTagger) error {
// If the something already exists where we are restoring to, archive existing file for versioning
// remove if it's a symlink, or fail if it's a directory
if info, err := dst.Lstat(filePath); err == nil {
switch {
case info.IsDir():
return errDirectory
case info.IsSymlink():
// Remove existing symlinks (as we don't want to archive them)
if err := dst.Remove(filePath); err != nil {
return errors.Wrap(err, "removing existing symlink")
}
case info.IsRegular():
if err := archiveFile(dst, src, filePath, tagger); err != nil {
return errors.Wrap(err, "archiving existing file")
}
default:
panic("bug: unknown item type")
}
} else if !fs.IsNotExist(err) {
return err
}
filePath = osutil.NativeFilename(filePath)
tag := versionTime.In(locationLocal).Truncate(time.Second).Format(TimeFormat)
taggedFilename := TagFilename(filePath, tag)
oldTaggedFilename := filePath + tag
untaggedFileName := filePath
// Check that the thing we've been asked to restore is actually a file
// and that it exists.
sourceFile := ""
for _, candidate := range []string{taggedFilename, oldTaggedFilename, untaggedFileName} {
if info, err := src.Lstat(candidate); fs.IsNotExist(err) || !info.IsRegular() {
continue
} else if err != nil {
// All other errors are fatal
return err
} else if candidate == untaggedFileName && !info.ModTime().Truncate(time.Second).Equal(versionTime) {
// No error, and untagged file, but mtime does not match, skip
continue
}
sourceFile = candidate
break
}
if sourceFile == "" {
return errNotFound
}
// Check that the target location of where we are supposed to restore does not exist.
// This should have been taken care of by the first few lines of this function.
if _, err := dst.Lstat(filePath); err == nil {
return errFileAlreadyExists
} else if !fs.IsNotExist(err) {
return err
}
_ = dst.MkdirAll(filepath.Dir(filePath), 0755)
return osutil.RenameOrCopy(src, dst, sourceFile, filePath)
}
func fsFromParams(folderFs fs.Filesystem, params map[string]string) (versionsFs fs.Filesystem) {
if params["fsType"] == "" && params["fsPath"] == "" {
versionsFs = fs.NewFilesystem(folderFs.Type(), filepath.Join(folderFs.URI(), ".stversions"))
} else if params["fsType"] == "" {
uri := params["fsPath"]
// We only know how to deal with relative folders for basic filesystems, as that's the only one we know
// how to check if it's absolute or relative.
if folderFs.Type() == fs.FilesystemTypeBasic && !filepath.IsAbs(params["fsPath"]) {
uri = filepath.Join(folderFs.URI(), params["fsPath"])
}
versionsFs = fs.NewFilesystem(folderFs.Type(), uri)
} else {
var fsType fs.FilesystemType
_ = fsType.UnmarshalText([]byte(params["fsType"]))
versionsFs = fs.NewFilesystem(fsType, params["fsPath"])
}
l.Debugln("%s (%s) folder using %s (%s) versioner dir", folderFs.URI(), folderFs.Type(), versionsFs.URI(), versionsFs.Type())
return
}

4
lib/versioner/versioner.go
View File

@ -9,6 +9,7 @@
package versioner
import (
"fmt"
"time"
"github.com/syncthing/syncthing/lib/fs"
@ -16,6 +17,8 @@ import (
type Versioner interface {
Archive(filePath string) error
GetVersions() (map[string][]FileVersion, error)
Restore(filePath string, versionTime time.Time) error
}
type FileVersion struct {
@ -25,6 +28,7 @@ type FileVersion struct {
}
var Factories = map[string]func(folderID string, filesystem fs.Filesystem, params map[string]string) Versioner{}
var ErrRestorationNotSupported = fmt.Errorf("version restoration not supported with the current versioner")
const (
TimeFormat = "20060102-150405"