Add trash can file versioning (fixes #1931)
This commit is contained in:
Jakob Borg
parent
983d7ec265
commit
a778763851
|
|
@ -589,6 +589,7 @@ func syncthingMain() {
|
|||
|
||||
m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb)
|
||||
cfg.Subscribe(m)
|
||||
mainSvc.Add(m)
|
||||
|
||||
if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {
|
||||
it, err := strconv.Atoi(t)
|
||||
|
|
|
|||
|
|
@ -263,6 +263,7 @@
|
|||
<tr ng-if="folder.versioning.type">
|
||||
<th><span class="glyphicon glyphicon-tags"></span> <span translate>File Versioning</span></th>
|
||||
<td class="text-right" ng-switch="folder.versioning.type">
|
||||
<span ng-switch-when="trashcan" translate>Trash Can File Versioning</span>
|
||||
<span ng-switch-when="staggered" translate>Staggered File Versioning</span>
|
||||
<span ng-switch-when="simple" translate>Simple File Versioning</span>
|
||||
<span ng-switch-when="external" translate>External File Versioning</span>
|
||||
|
|
@ -681,27 +682,27 @@
|
|||
</select>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label translate>File Versioning</label>
|
||||
<div class="radio">
|
||||
<label>
|
||||
<input type="radio" ng-model="currentFolder.fileVersioningSelector" value="none"> <span translate>No File Versioning</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="radio">
|
||||
<label>
|
||||
<input type="radio" ng-model="currentFolder.fileVersioningSelector" value="simple"> <span translate>Simple File Versioning</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="radio">
|
||||
<label>
|
||||
<input type="radio" ng-model="currentFolder.fileVersioningSelector" value="staggered"> <span translate>Staggered File Versioning</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="radio">
|
||||
<label>
|
||||
<input type="radio" ng-model="currentFolder.fileVersioningSelector" value="external"> <span translate>External File Versioning</span>
|
||||
</label>
|
||||
<label translate>File Versioning</label> <a href="http://docs.syncthing.net/users/versioning.html" target="_blank"><span class="glyphicon glyphicon-book"></span> <span translate>Help</span></a>
|
||||
<select class="form-control" ng-model="currentFolder.fileVersioningSelector">
|
||||
<option value="none" translate>No File Versioning</option>
|
||||
<option value="trashcan" translate>Trash Can File Versioning</option>
|
||||
<option value="simple" translate>Simple File Versioning</option>
|
||||
<option value="staggered" translate>Staggered File Versioning</option>
|
||||
<option value="external" translate>External File Versioning</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" ng-if="currentFolder.fileVersioningSelector=='trashcan'" ng-class="{'has-error': folderEditor.trashcanClean.$invalid && folderEditor.trashcanClean.$dirty}">
|
||||
<p translate class="help-block">Files are moved to .stversions folder when replaced or deleted by Syncthing.</p>
|
||||
<label translate for="trashcanClean">Clean out after</label>
|
||||
<div class="input-group">
|
||||
<input name="trashcanClean" id="trashcanClean" class="form-control text-right" type="number" ng-model="currentFolder.trashcanClean" required min="0"></input>
|
||||
<div class="input-group-addon">days</div>
|
||||
</div>
|
||||
<p class="help-block">
|
||||
<span translate ng-if="folderEditor.trashcanClean.$valid || folderEditor.trashcanClean.$pristine">The number of days to keep files in the trash can. Zero means forever.</span>
|
||||
<span translate ng-if="folderEditor.trashcanClean.$error.required && folderEditor.trashcanClean.$dirty">The number of days must be a number and cannot be blank.</span>
|
||||
<span translate ng-if="folderEditor.trashcanClean.$error.min && folderEditor.trashcanClean.$dirty">A negative number of days doesn't make sense.</span>
|
||||
</p>
|
||||
</div>
|
||||
<div class="form-group" ng-if="currentFolder.fileVersioningSelector=='simple'" ng-class="{'has-error': folderEditor.simpleKeep.$invalid && folderEditor.simpleKeep.$dirty}">
|
||||
<p translate class="help-block">Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.</p>
|
||||
|
|
|
|||
|
|
@ -521,10 +521,10 @@ angular.module('syncthing.core')
|
|||
if ($scope.model[folderCfg.id].state == 'error') {
|
||||
return 'stopped'; // legacy, the state is called "stopped" in the GUI
|
||||
}
|
||||
|
||||
|
||||
// after restart syncthing process state may be empty
|
||||
if (!$scope.model[folderCfg.id].state) {
|
||||
return 'unknown';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
return '' + $scope.model[folderCfg.id].state;
|
||||
|
|
@ -990,7 +990,11 @@ angular.module('syncthing.core')
|
|||
$scope.currentFolder.devices.forEach(function (n) {
|
||||
$scope.currentFolder.selectedDevices[n.deviceID] = true;
|
||||
});
|
||||
if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "simple") {
|
||||
if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "trashcan") {
|
||||
$scope.currentFolder.trashcanFileVersioning = true;
|
||||
$scope.currentFolder.fileVersioningSelector = "trashcan";
|
||||
$scope.currentFolder.trashcanClean = +$scope.currentFolder.versioning.params.cleanoutDays;
|
||||
} else if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "simple") {
|
||||
$scope.currentFolder.simpleFileVersioning = true;
|
||||
$scope.currentFolder.fileVersioningSelector = "simple";
|
||||
$scope.currentFolder.simpleKeep = +$scope.currentFolder.versioning.params.keep;
|
||||
|
|
@ -1007,6 +1011,7 @@ angular.module('syncthing.core')
|
|||
} else {
|
||||
$scope.currentFolder.fileVersioningSelector = "none";
|
||||
}
|
||||
$scope.currentFolder.trashcanClean = $scope.currentFolder.trashcanClean || 0; // weeds out nulls and undefineds
|
||||
$scope.currentFolder.simpleKeep = $scope.currentFolder.simpleKeep || 5;
|
||||
$scope.currentFolder.staggeredCleanInterval = $scope.currentFolder.staggeredCleanInterval || 3600;
|
||||
$scope.currentFolder.staggeredVersionsPath = $scope.currentFolder.staggeredVersionsPath || "";
|
||||
|
|
@ -1030,6 +1035,7 @@ angular.module('syncthing.core')
|
|||
};
|
||||
$scope.currentFolder.rescanIntervalS = 60;
|
||||
$scope.currentFolder.fileVersioningSelector = "none";
|
||||
$scope.currentFolder.trashcanClean = 0;
|
||||
$scope.currentFolder.simpleKeep = 5;
|
||||
$scope.currentFolder.staggeredMaxAge = 365;
|
||||
$scope.currentFolder.staggeredCleanInterval = 3600;
|
||||
|
|
@ -1051,6 +1057,7 @@ angular.module('syncthing.core')
|
|||
|
||||
$scope.currentFolder.rescanIntervalS = 60;
|
||||
$scope.currentFolder.fileVersioningSelector = "none";
|
||||
$scope.currentFolder.trashcanClean = 0;
|
||||
$scope.currentFolder.simpleKeep = 5;
|
||||
$scope.currentFolder.staggeredMaxAge = 365;
|
||||
$scope.currentFolder.staggeredCleanInterval = 3600;
|
||||
|
|
@ -1087,7 +1094,16 @@ angular.module('syncthing.core')
|
|||
}
|
||||
delete folderCfg.selectedDevices;
|
||||
|
||||
if (folderCfg.fileVersioningSelector === "simple") {
|
||||
if (folderCfg.fileVersioningSelector === "trashcan") {
|
||||
folderCfg.versioning = {
|
||||
'Type': 'trashcan',
|
||||
'Params': {
|
||||
'cleanoutDays': '' + folderCfg.trashcanClean
|
||||
}
|
||||
};
|
||||
delete folderCfg.trashcanFileVersioning;
|
||||
delete folderCfg.trashcanClean;
|
||||
} else if (folderCfg.fileVersioningSelector === "simple") {
|
||||
folderCfg.versioning = {
|
||||
'Type': 'simple',
|
||||
'Params': {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -35,6 +35,7 @@ import (
|
|||
"github.com/syncthing/syncthing/internal/sync"
|
||||
"github.com/syncthing/syncthing/internal/versioner"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/thejerf/suture"
|
||||
)
|
||||
|
||||
// How many files to send in each Index/IndexUpdate message.
|
||||
|
|
@ -61,6 +62,8 @@ type service interface {
|
|||
}
|
||||
|
||||
type Model struct {
|
||||
*suture.Supervisor
|
||||
|
||||
cfg *config.Wrapper
|
||||
db *leveldb.DB
|
||||
finder *db.BlockFinder
|
||||
|
|
@ -103,6 +106,13 @@ var (
|
|||
// for file data without altering the local folder in any way.
|
||||
func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName, clientVersion string, ldb *leveldb.DB) *Model {
|
||||
m := &Model{
|
||||
Supervisor: suture.New("model", suture.Spec{
|
||||
Log: func(line string) {
|
||||
if debug {
|
||||
l.Debugln(line)
|
||||
}
|
||||
},
|
||||
}),
|
||||
cfg: cfg,
|
||||
db: ldb,
|
||||
finder: db.NewBlockFinder(ldb, cfg),
|
||||
|
|
@ -168,7 +178,14 @@ func (m *Model) StartFolderRW(folder string) {
|
|||
if !ok {
|
||||
l.Fatalf("Requested versioning type %q that does not exist", cfg.Versioning.Type)
|
||||
}
|
||||
p.versioner = factory(folder, cfg.Path(), cfg.Versioning.Params)
|
||||
versioner := factory(folder, cfg.Path(), cfg.Versioning.Params)
|
||||
if service, ok := versioner.(suture.Service); ok {
|
||||
// The versioner implements the suture.Service interface, so
|
||||
// expects to be run in the background in addition to being called
|
||||
// when files are going to be archived.
|
||||
m.Add(service)
|
||||
}
|
||||
p.versioner = versioner
|
||||
}
|
||||
|
||||
go p.Serve()
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
testdata
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
// Copyright (C) 2015 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package versioner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/internal/osutil"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Register the constructor for this type of versioner
|
||||
Factories["trashcan"] = NewTrashcan
|
||||
}
|
||||
|
||||
type Trashcan struct {
|
||||
folderPath string
|
||||
cleanoutDays int
|
||||
stop chan struct{}
|
||||
}
|
||||
|
||||
func NewTrashcan(folderID, folderPath string, params map[string]string) Versioner {
|
||||
cleanoutDays, _ := strconv.Atoi(params["cleanoutDays"])
|
||||
// On error we default to 0, "do not clean out the trash can"
|
||||
|
||||
s := &Trashcan{
|
||||
folderPath: folderPath,
|
||||
cleanoutDays: cleanoutDays,
|
||||
stop: make(chan struct{}),
|
||||
}
|
||||
|
||||
if debug {
|
||||
l.Debugf("instantiated %#v", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Archive moves the named file away to a version archive. If this function
|
||||
// returns nil, the named file does not exist any more (has been archived).
|
||||
func (t *Trashcan) Archive(filePath string) error {
|
||||
_, err := osutil.Lstat(filePath)
|
||||
if os.IsNotExist(err) {
|
||||
if debug {
|
||||
l.Debugln("not archiving nonexistent file", filePath)
|
||||
}
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionsDir := filepath.Join(t.folderPath, ".stversions")
|
||||
if _, err := os.Stat(versionsDir); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
if debug {
|
||||
l.Debugln("creating versions dir", versionsDir)
|
||||
}
|
||||
if err := osutil.MkdirAll(versionsDir, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
osutil.HideFile(versionsDir)
|
||||
}
|
||||
|
||||
if debug {
|
||||
l.Debugln("archiving", filePath)
|
||||
}
|
||||
|
||||
relativePath, err := filepath.Rel(t.folderPath, filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
archivedPath := filepath.Join(versionsDir, relativePath)
|
||||
if err := osutil.MkdirAll(filepath.Dir(archivedPath), 0777); err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
if debug {
|
||||
l.Debugln("moving to", archivedPath)
|
||||
}
|
||||
|
||||
if err := osutil.Rename(filePath, archivedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set the mtime to the time the file was deleted. This is used by the
|
||||
// cleanout routine. If this fails things won't work optimally but there's
|
||||
// not much we can do about it so we ignore the error.
|
||||
os.Chtimes(archivedPath, time.Now(), time.Now())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Trashcan) Serve() {
|
||||
if debug {
|
||||
l.Debugln(t, "starting")
|
||||
defer l.Debugln(t, "stopping")
|
||||
}
|
||||
|
||||
// Do the first cleanup one minute after startup.
|
||||
timer := time.NewTimer(time.Minute)
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-t.stop:
|
||||
return
|
||||
|
||||
case <-timer.C:
|
||||
if t.cleanoutDays > 0 {
|
||||
if err := t.cleanoutArchive(); err != nil {
|
||||
l.Infoln("Cleaning trashcan:", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanups once a day should be enough.
|
||||
timer.Reset(24 * time.Hour)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Trashcan) Stop() {
|
||||
close(t.stop)
|
||||
}
|
||||
|
||||
func (t *Trashcan) String() string {
|
||||
return fmt.Sprintf("trashcan@%p", t)
|
||||
}
|
||||
|
||||
func (t *Trashcan) cleanoutArchive() error {
|
||||
versionsDir := filepath.Join(t.folderPath, ".stversions")
|
||||
if _, err := osutil.Lstat(versionsDir); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(time.Duration(-24*t.cleanoutDays) * time.Hour)
|
||||
currentDir := ""
|
||||
filesInDir := 0
|
||||
walkFn := func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
// We have entered a new directory. Lets check if the previous
|
||||
// directory was empty and try to remove it. We ignore failure for
|
||||
// the time being.
|
||||
if currentDir != "" && filesInDir == 0 {
|
||||
osutil.Remove(currentDir)
|
||||
}
|
||||
currentDir = path
|
||||
filesInDir = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.ModTime().Before(cutoff) {
|
||||
// The file is too old; remove it.
|
||||
osutil.Remove(path)
|
||||
} else {
|
||||
// Keep this file, and remember it so we don't unnecessarily try
|
||||
// to remove this directory.
|
||||
filesInDir++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := filepath.Walk(versionsDir, walkFn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The last directory seen by the walkFn may not have been removed as it
|
||||
// should be.
|
||||
if currentDir != "" && filesInDir == 0 {
|
||||
osutil.Remove(currentDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
// Copyright (C) 2015 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
package versioner
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTrashcanCleanout(t *testing.T) {
|
||||
// Verify that files older than the cutoff are removed, that files newer
|
||||
// than the cutoff are *not* removed, and that empty directories are
|
||||
// removed (best effort).
|
||||
|
||||
var testcases = []struct {
|
||||
file string
|
||||
shouldRemove bool
|
||||
}{
|
||||
{"testdata/.stversions/file1", false},
|
||||
{"testdata/.stversions/file2", true},
|
||||
{"testdata/.stversions/keep1/file1", false},
|
||||
{"testdata/.stversions/keep1/file2", false},
|
||||
{"testdata/.stversions/keep2/file1", false},
|
||||
{"testdata/.stversions/keep2/file2", true},
|
||||
{"testdata/.stversions/remove/file1", true},
|
||||
{"testdata/.stversions/remove/file2", true},
|
||||
}
|
||||
|
||||
os.RemoveAll("testdata")
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
oldTime := time.Now().Add(-8 * 24 * time.Hour)
|
||||
for _, tc := range testcases {
|
||||
os.MkdirAll(filepath.Dir(tc.file), 0777)
|
||||
if err := ioutil.WriteFile(tc.file, []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.shouldRemove {
|
||||
if err := os.Chtimes(tc.file, oldTime, oldTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
versioner := NewTrashcan("default", "testdata", map[string]string{"cleanoutDays": "7"}).(*Trashcan)
|
||||
if err := versioner.cleanoutArchive(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
_, err := os.Lstat(tc.file)
|
||||
if tc.shouldRemove && !os.IsNotExist(err) {
|
||||
t.Error(tc.file, "should have been removed")
|
||||
} else if !tc.shouldRemove && err != nil {
|
||||
t.Error(tc.file, "should not have been removed")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := os.Lstat("testdata/.stversions/remove"); !os.IsNotExist(err) {
|
||||
t.Error("empty directory should have been removed")
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue