Simple file versioning (fixes #218)

This commit is contained in:
Jakob Borg 2014-05-25 20:49:08 +02:00
parent dd971b56e5
commit 3d055bbb79
17 changed files with 350 additions and 120 deletions

File diff suppressed because one or more lines are too long

View File

@ -24,6 +24,7 @@ import (
"github.com/calmh/syncthing/discover"
"github.com/calmh/syncthing/logger"
"github.com/calmh/syncthing/model"
"github.com/calmh/syncthing/osutil"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/upnp"
"github.com/juju/ratelimit"
@ -498,7 +499,7 @@ func saveConfigLoop(cfgFile string) {
continue
}
err = model.Rename(cfgFile+".tmp", cfgFile)
err = osutil.Rename(cfgFile+".tmp", cfgFile)
if err != nil {
l.Warnln(err)
}

View File

@ -27,13 +27,56 @@ type Configuration struct {
}
type RepositoryConfiguration struct {
ID string `xml:"id,attr"`
Directory string `xml:"directory,attr"`
Nodes []NodeConfiguration `xml:"node"`
ReadOnly bool `xml:"ro,attr"`
IgnorePerms bool `xml:"ignorePerms,attr"`
Invalid string `xml:"-"` // Set at runtime when there is an error, not saved
nodeIDs []string
ID string `xml:"id,attr"`
Directory string `xml:"directory,attr"`
Nodes []NodeConfiguration `xml:"node"`
ReadOnly bool `xml:"ro,attr"`
IgnorePerms bool `xml:"ignorePerms,attr"`
Invalid string `xml:"-"` // Set at runtime when there is an error, not saved
Versioning VersioningConfiguration `xml:"versioning"`
nodeIDs []string
}
type VersioningConfiguration struct {
Type string `xml:"type,attr"`
Params map[string]string
}
type InternalVersioningConfiguration struct {
Type string `xml:"type,attr,omitempty"`
Params []InternalParam `xml:"param"`
}
type InternalParam struct {
Key string `xml:"key,attr"`
Val string `xml:"val,attr"`
}
func (c *VersioningConfiguration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
var tmp InternalVersioningConfiguration
tmp.Type = c.Type
for k, v := range c.Params {
tmp.Params = append(tmp.Params, InternalParam{k, v})
}
return e.EncodeElement(tmp, start)
}
func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var tmp InternalVersioningConfiguration
err := d.DecodeElement(&tmp, &start)
if err != nil {
return err
}
c.Type = tmp.Type
c.Params = make(map[string]string, len(tmp.Params))
for _, p := range tmp.Params {
c.Params[p.Key] = p.Val
}
return nil
}
func (r *RepositoryConfiguration) NodeIDs() []string {

View File

@ -405,10 +405,16 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
}
$scope.editRepo = function (nodeCfg) {
$scope.currentRepo = $.extend({selectedNodes: {}}, nodeCfg);
$scope.currentRepo = angular.copy(nodeCfg);
$scope.currentRepo.selectedNodes = {};
$scope.currentRepo.Nodes.forEach(function (n) {
$scope.currentRepo.selectedNodes[n.NodeID] = true;
});
if ($scope.currentRepo.Versioning && $scope.currentRepo.Versioning.Type === "simple") {
$scope.currentRepo.simpleFileVersioning = true;
$scope.currentRepo.simpleKeep = +$scope.currentRepo.Versioning.Params.keep;
}
$scope.currentRepo.simpleKeep = $scope.currentRepo.simpleKeep || 5;
$scope.editingExisting = true;
$scope.repoEditor.$setPristine();
$('#editRepo').modal({backdrop: 'static', keyboard: true});
@ -436,6 +442,19 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
}
delete repoCfg.selectedNodes;
if (repoCfg.simpleFileVersioning) {
repoCfg.Versioning = {
'Type': 'simple',
'Params': {
'keep': '' + repoCfg.simpleKeep,
}
};
delete repoCfg.simpleFileVersioning;
delete repoCfg.simpleKeep;
} else {
delete repoCfg.Versioning;
}
$scope.repos[repoCfg.ID] = repoCfg;
$scope.config.Repositories = repoList($scope.repos);

View File

@ -455,47 +455,75 @@
</div>
<div class="modal-body">
<form role="form" name="repoEditor">
<div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
<label for="repoID">Repository ID</label>
<input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
<p class="help-block">
<span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
<span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
<span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
</p>
</div>
<div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
<label for="repoPath">Repository Path</label>
<input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
<p class="help-block">
<span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist. The tilde character <code>~</code> can be used as a shortcut for <code>{{system.tilde}}</code>.</span>
<span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
</p>
</div>
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
</label>
<div class="row">
<div class="col-md-12">
<div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
<label for="repoID">Repository ID</label>
<input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
<p class="help-block">
<span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
<span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
<span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
</p>
</div>
<div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
<label for="repoPath">Repository Path</label>
<input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
<p class="help-block">
<span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist. The tilde character <code>~</code> can be used as a shortcut for <code>{{system.tilde}}</code>.</span>
<span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
</p>
</div>
</div>
<p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
</div>
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" ng-model="currentRepo.IgnorePerms"> Ignore Permissions
</label>
<div class="row">
<div class="col-md-6">
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
</label>
</div>
<p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
</div>
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" ng-model="currentRepo.IgnorePerms"> Ignore Permissions
</label>
</div>
<p class="help-block">File permission bits are ignored when looking for changes. Use on FAT filesystems.</p>
</div>
<div class="form-group">
<label for="nodes">Nodes</label>
<div class="checkbox" ng-repeat="node in otherNodes()">
<label>
<input type="checkbox" ng-model="currentRepo.selectedNodes[node.NodeID]"> {{nodeName(node)}}
</label>
</div>
<p class="help-block">Select the nodes to share this repository with.</p>
</div>
</div>
<p class="help-block">File permission bits are ignored when looking for changes. Use on FAT filesystems.</p>
</div>
<div class="form-group">
<label for="nodes">Nodes</label>
<div class="checkbox" ng-repeat="node in otherNodes()">
<label>
<input type="checkbox" ng-model="currentRepo.selectedNodes[node.NodeID]"> {{nodeName(node)}}
</label>
<div class="col-md-6">
<div class="form-group">
<div class="checkbox">
<label>
<input type="checkbox" ng-model="currentRepo.simpleFileVersioning"> Simple File Versioning
</label>
</div>
<p class="help-block">Files are moved to date stamped versions in a <code>.stversions</code> folder when replaced or deleted by syncthing.</p>
</div>
<div class="form-group" ng-if="currentRepo.simpleFileVersioning" ng-class="{'has-error': repoEditor.simpleKeep.$invalid && repoEditor.simpleKeep.$dirty}">
<label for="simpleKeep">Keep Versions</label>
<input name="simpleKeep" id="simpleKeep" class="form-control" type="number" ng-model="currentRepo.simpleKeep" required min="1"></input>
<p class="help-block">
<span ng-if="repoEditor.simpleKeep.$valid || repoEditor.simpleKeep.$pristine">The number of old versions to keep, per file.</span>
<span ng-if="repoEditor.simpleKeep.$error.required && repoEditor.simpleKeep.$dirty">The number of versions must be a number and cannot be blank.</span>
<span ng-if="repoEditor.simpleKeep.$error.min && repoEditor.simpleKeep.$dirty">You must keep at least one version.</span>
</p>
</div>
</div>
<p class="help-block">Select the nodes to share this repository with.</p>
</div>
</form>
<div ng-show="!editingExisting">

View File

@ -17,6 +17,7 @@ import (
"github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/osutil"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
@ -716,7 +717,7 @@ func (m *Model) saveIndex(repo string, dir string, fs []protocol.FileInfo) {
gzw.Close()
idxf.Close()
Rename(name+".tmp", name)
osutil.Rename(name+".tmp", name)
}
func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {

View File

@ -10,8 +10,10 @@ import (
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/cid"
"github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/osutil"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/calmh/syncthing/versioner"
)
type requestResult struct {
@ -71,6 +73,7 @@ type puller struct {
requestSlots chan bool
blocks chan bqBlock
requestResults chan requestResult
versioner versioner.Versioner
}
func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int, cfg *config.Configuration) *puller {
@ -86,6 +89,14 @@ func newPuller(repoCfg config.RepositoryConfiguration, model *Model, slots int,
requestResults: make(chan requestResult),
}
if len(repoCfg.Versioning.Type) > 0 {
factory, ok := versioner.Factories[repoCfg.Versioning.Type]
if !ok {
l.Fatalf("Requested versioning type %q that does not exist", repoCfg.Versioning.Type)
}
p.versioner = factory(repoCfg.Versioning.Params)
}
if slots > 0 {
// Read/write
for i := 0; i < slots; i++ {
@ -221,6 +232,10 @@ func (p *puller) fixupDirectories() {
return nil
}
if filepath.Base(rn) == ".stversions" {
return nil
}
cur := p.model.CurrentRepoFile(p.repoCfg.ID, rn)
if cur.Name != rn {
// No matching dir in current list; weird
@ -284,10 +299,10 @@ func (p *puller) fixupDirectories() {
l.Debugln("delete dir:", dir)
}
err := os.Remove(dir)
if err != nil {
l.Warnln(err)
} else {
if err == nil {
deleted++
} else if p.versioner == nil { // Failures are expected in the presence of versioning
l.Warnln(err)
}
}
@ -385,7 +400,7 @@ func (p *puller) handleBlock(b bqBlock) bool {
}
return true
}
defTempNamer.Hide(of.temp)
osutil.HideFile(of.temp)
}
if of.err != nil {
@ -524,7 +539,11 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
}
os.Remove(of.temp)
os.Chmod(of.filepath, 0666)
if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
if p.versioner != nil {
if err := p.versioner.Archive(of.filepath); err == nil {
p.model.updateLocal(p.repoCfg.ID, f)
}
} else if err := os.Remove(of.filepath); err == nil || os.IsNotExist(err) {
p.model.updateLocal(p.repoCfg.ID, f)
}
} else {
@ -540,8 +559,8 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
delete(p.openFiles, f.Name)
return
}
defTempNamer.Show(of.temp)
if Rename(of.temp, of.filepath) == nil {
osutil.ShowFile(of.temp)
if osutil.Rename(of.temp, of.filepath) == nil {
p.model.updateLocal(p.repoCfg.ID, f)
}
}
@ -614,11 +633,23 @@ func (p *puller) closeFile(f scanner.File) {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
}
defTempNamer.Show(of.temp)
osutil.ShowFile(of.temp)
if p.versioner != nil {
err := p.versioner.Archive(of.filepath)
if err != nil {
if debug {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
}
return
}
}
if debug {
l.Debugf("pull: rename %q / %q: %q", p.repoCfg.ID, f.Name, of.filepath)
}
if err := Rename(of.temp, of.filepath); err == nil {
if err := osutil.Rename(of.temp, of.filepath); err == nil {
p.model.updateLocal(p.repoCfg.ID, f)
} else {
l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)

View File

@ -23,11 +23,3 @@ func (t tempNamer) TempName(name string) string {
tname := fmt.Sprintf("%s.%s", t.prefix, filepath.Base(name))
return filepath.Join(tdir, tname)
}
func (t tempNamer) Hide(path string) error {
return nil
}
func (t tempNamer) Show(path string) error {
return nil
}

View File

@ -6,7 +6,6 @@ import (
"fmt"
"path/filepath"
"strings"
"syscall"
)
type tempNamer struct {
@ -24,33 +23,3 @@ func (t tempNamer) TempName(name string) string {
tname := fmt.Sprintf("%s.%s.tmp", t.prefix, filepath.Base(name))
return filepath.Join(tdir, tname)
}
func (t tempNamer) Hide(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func (t tempNamer) Show(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}

View File

@ -2,26 +2,12 @@ package model
import (
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
func Rename(from, to string) error {
if runtime.GOOS == "windows" {
os.Chmod(to, 0666) // Make sure the file is user writeable
err := os.Remove(to)
if err != nil && !os.IsNotExist(err) {
l.Warnln(err)
}
}
defer os.Remove(from) // Don't leave a dangling temp file in case of rename error
return os.Rename(from, to)
}
func fileFromFileInfo(f protocol.FileInfo) scanner.File {
var blocks = make([]scanner.Block, len(f.Blocks))
var offset int64

11
osutil/hidden_unix.go Normal file
View File

@ -0,0 +1,11 @@
// +build !windows
package osutil
func HideFile(path string) error {
return nil
}
func ShowFile(path string) error {
return nil
}

35
osutil/hidden_windows.go Normal file
View File

@ -0,0 +1,35 @@
// +build windows
package osutil
import "syscall"
func HideFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func ShowFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}

18
osutil/osutil.go Normal file
View File

@ -0,0 +1,18 @@
package osutil
import (
"os"
"runtime"
)
func Rename(from, to string) error {
if runtime.GOOS == "windows" {
os.Chmod(to, 0666) // Make sure the file is user writeable
err := os.Remove(to)
if err != nil && !os.IsNotExist(err) {
return err
}
}
defer os.Remove(from) // Don't leave a dangling temp file in case of rename error
return os.Rename(from, to)
}

View File

@ -144,15 +144,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
return nil
}
if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
// An ignore-file; these are ignored themselves
if debug {
l.Debugln("ignorefile:", rn)
}
return nil
}
if w.ignoreFile(ign, rn) {
if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
// An ignored file
if debug {
l.Debugln("ignored:", rn)

13
versioner/debug.go Normal file
View File

@ -0,0 +1,13 @@
package versioner
import (
"os"
"strings"
"github.com/calmh/syncthing/logger"
)
var (
debug = strings.Contains(os.Getenv("STTRACE"), "versioner") || os.Getenv("STTRACE") == "all"
l = logger.DefaultLogger
)

84
versioner/simple.go Normal file
View File

@ -0,0 +1,84 @@
package versioner
import (
"os"
"path/filepath"
"sort"
"strconv"
"time"
"github.com/calmh/syncthing/osutil"
)
func init() {
// Register the constructor for this type of versioner with the name "simple"
Factories["simple"] = NewSimple
}
// The type holds our configuration
type Simple struct {
keep int
}
// The constructor function takes a map of parameters and creates the type.
func NewSimple(params map[string]string) Versioner {
keep, err := strconv.Atoi(params["keep"])
if err != nil {
keep = 5 // A reasonable default
}
s := Simple{
keep: keep,
}
if debug {
l.Debugf("instantiated %#v", s)
}
return s
}
// Move away the named file to a version archive. If this function returns
// nil, the named file does not exist any more (has been archived).
func (v Simple) Archive(path string) error {
_, err := os.Stat(path)
if err != nil && os.IsNotExist(err) {
return nil
}
if debug {
l.Debugln("archiving", path)
}
file := filepath.Base(path)
dir := filepath.Join(filepath.Dir(path), ".stversions")
err = os.MkdirAll(dir, 0755)
if err != nil && !os.IsExist(err) {
return err
} else {
osutil.HideFile(dir)
}
ver := file + "~" + time.Now().Format("20060102-150405")
err = osutil.Rename(path, filepath.Join(dir, ver))
if err != nil {
return err
}
versions, err := filepath.Glob(filepath.Join(dir, file+"~*"))
if err != nil {
l.Warnln(err)
return nil
}
if len(versions) > v.keep {
sort.Strings(versions)
for _, toRemove := range versions[:len(versions)-v.keep] {
err = os.Remove(toRemove)
if err != nil {
l.Warnln(err)
}
}
}
return nil
}

7
versioner/versioner.go Normal file
View File

@ -0,0 +1,7 @@
package versioner
type Versioner interface {
Archive(path string) error
}
var Factories = map[string]func(map[string]string) Versioner{}