Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Add receive only folder type (#5027) 

Adds a receive only folder type that does not send changes, and where the user can optionally revert local changes. Also changes some of the icons to make the three folder types distinguishable.
This commit is contained in:
Jakob Borg 2018-07-12 11:15:57 +03:00 committed by GitHub
parent 1a6c7587c2
commit f822b10550
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1136 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/syncthing/gui.go
View File

@ -85,6 +85,7 @@ type modelIntf interface {
GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
Completion(device protocol.DeviceID, folder string) model.FolderCompletion
Override(folder string)
Revert(folder string)
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
NeedSize(folder string) db.Counts
@ -107,6 +108,7 @@ type modelIntf interface {
Connection(deviceID protocol.DeviceID) (connections.Connection, bool)
GlobalSize(folder string) db.Counts
LocalSize(folder string) db.Counts
ReceiveOnlyChangedSize(folder string) db.Counts
CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool)
State(folder string) (string, time.Time, error)
@ -293,6 +295,7 @@ func (s *apiService) Serve() {
postRestMux.HandleFunc("/rest/db/prio", s.postDBPrio) // folder file [perpage] [page]
postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores) // folder
postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder
postRestMux.HandleFunc("/rest/db/revert", s.postDBRevert) // folder
postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay]
postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore) // folder <body>
postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // <body>
@ -712,6 +715,17 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter
need := m.NeedSize(folder)
res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes
if cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
// Add statistics for things that have changed locally in a receive
// only folder.
ro := m.ReceiveOnlyChangedSize(folder)
res["receiveOnlyChangedFiles"] = ro.Files
res["receiveOnlyChangedDirectories"] = ro.Directories
res["receiveOnlyChangedSymlinks"] = ro.Symlinks
res["receiveOnlyChangedDeletes"] = ro.Deleted
res["receiveOnlyChangedBytes"] = ro.Bytes
}
res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes
res["state"], res["stateChanged"], err = m.State(folder)
@ -748,6 +762,12 @@ func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) {
go s.model.Override(folder)
}
func (s *apiService) postDBRevert(w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var folder = qs.Get("folder")
go s.model.Revert(folder)
}
func getPagingParams(qs url.Values) (int, int) {
page, err := strconv.Atoi(qs.Get("page"))
if err != nil || page < 1 {

6
cmd/syncthing/mocked_model_test.go
View File

@ -29,6 +29,8 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model.
func (m *mockedModel) Override(folder string) {}
func (m *mockedModel) Revert(folder string) {}
func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
return nil, nil, nil
}
@ -117,6 +119,10 @@ func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {
return 0, false
}

10
gui/default/index.html
View File

@ -301,7 +301,9 @@
<div class="panel-progress" ng-show="folderStatus(folder) == 'scanning' && scanProgress[folder.id] != undefined" ng-attr-style="width: {{scanPercentage(folder.id) | percent}}"></div>
<h4 class="panel-title">
<div class="panel-icon hidden-xs">
<span ng-class="[folder.type == 'sendonly' ? 'fas fa-fw fa-lock' : 'fas fa-fw fa-folder']"></span>
<span ng-if="folder.type == 'sendreceive'" class="fas fa-fw fa-folder"></span>
<span ng-if="folder.type == 'sendonly'" class="fas fa-fw fa-upload"></span>
<span ng-if="folder.type == 'receiveonly'" class="fas fa-fw fa-download"></span>
</div>
<div class="panel-status pull-right text-{{folderClass(folder)}}" ng-switch="folderStatus(folder)">
<span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs">&#9724;</span></span>
@ -386,9 +388,10 @@
</td>
</tr>
<tr ng-if="folder.type != 'sendreceive'">
<th><span class="fas fa-fw fa-lock"></span>&nbsp;<span translate>Folder Type</span></th>
<th><span class="fas fa-fw fa-folder"></span>&nbsp;<span translate>Folder Type</span></th>
<td class="text-right">
<span ng-if="folder.type == 'sendonly'" translate>Send Only</span>
<span ng-if="folder.type == 'receiveonly'" translate>Receive Only</span>
</td>
</tr>
<tr ng-if="folder.ignorePerms">
@ -478,6 +481,9 @@
<button type="button" class="btn btn-sm btn-danger pull-left" ng-click="override(folder.id)" ng-if="folderStatus(folder) == 'outofsync' && folder.type == 'sendonly'">
<span class="fas fa-arrow-circle-up"></span>&nbsp;<span translate>Override Changes</span>
</button>
<button type="button" class="btn btn-sm btn-danger pull-left" ng-click="revert(folder.id)" ng-if="canRevert(folder.id)">
<span class="fa fa-arrow-circle-down"></span>&nbsp;<span translate>Revert Local Changes</span>
</button>
<span class="pull-right">
<button ng-if="!folder.paused" type="button" class="btn btn-sm btn-default" ng-click="setFolderPause(folder.id, true)">
<span class="fas fa-pause"></span>&nbsp;<span translate>Pause</span>

16
gui/default/syncthing/core/syncthingController.js
View File

@ -2102,6 +2102,22 @@ angular.module('syncthing.core')
$http.post(urlbase + "/db/override?folder=" + encodeURIComponent(folder));
};
$scope.revert = function (folder) {
$http.post(urlbase + "/db/revert?folder=" + encodeURIComponent(folder));
};
$scope.canRevert = function (folder) {
var f = $scope.model[folder];
if (!f) {
return false;
}
return f.receiveOnlyChangedBytes > 0 ||
f.receiveOnlyChangedDeletes > 0 ||
f.receiveOnlyChangedDirectories > 0 ||
f.receiveOnlyChangedFiles > 0 ||
f.receiveOnlyChangedSymlinks > 0;
};
$scope.advanced = function () {
$scope.advancedConfig = angular.copy($scope.config);
$('#advanced').modal('show');

2
gui/default/syncthing/folder/editFolderModalView.html
View File

@ -162,8 +162,10 @@
<select class="form-control" ng-model="currentFolder.type">
<option value="sendreceive" translate>Send &amp; Receive</option>
<option value="sendonly" translate>Send Only</option>
<option value="receiveonly" translate>Receive Only</option>
</select>
<p ng-if="currentFolder.type == 'sendonly'" translate class="help-block">Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.</p>
<p ng-if="currentFolder.type == 'receiveonly'" translate class="help-block">Files are synchronized from the cluster, but any changes made locally will not be sent to other devices.</p>
</div>
<div class="col-md-6 form-group">
<label translate>File Pull Order</label>

5
lib/config/foldertype.go
View File

@ -11,6 +11,7 @@ type FolderType int
const (
FolderTypeSendReceive FolderType = iota // default is sendreceive
FolderTypeSendOnly
FolderTypeReceiveOnly
)
func (t FolderType) String() string {
@ -19,6 +20,8 @@ func (t FolderType) String() string {
return "sendreceive"
case FolderTypeSendOnly:
return "sendonly"
case FolderTypeReceiveOnly:
return "receiveonly"
default:
return "unknown"
}
@ -34,6 +37,8 @@ func (t *FolderType) UnmarshalText(bs []byte) error {
*t = FolderTypeSendReceive
case "readonly", "sendonly":
*t = FolderTypeSendOnly
case "receiveonly":
*t = FolderTypeReceiveOnly
default:
*t = FolderTypeSendReceive
}

2
lib/db/leveldb_dbinstance.go
View File

@ -586,7 +586,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
if i == 0 {
if fi, ok := db.getFile(fk); ok {
meta.addFile(globalDeviceID, fi)
meta.addFile(protocol.GlobalDeviceID, fi)
}
}
}

4
lib/db/leveldb_test.go
View File

@ -305,7 +305,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Unexpected additional file via sequence", f.FileName())
return true
}
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) {
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) {
found = true
} else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
@ -330,7 +330,7 @@ func TestUpdate0to3(t *testing.T) {
}
f := fi.(protocol.FileInfo)
delete(need, f.Name)
if !f.IsEquivalent(e, true, true) {
if !f.IsEquivalentOptional(e, true, true, 0) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
return true

8
lib/db/leveldb_transactions.go
View File

@ -140,11 +140,11 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
meta.removeFile(globalDeviceID, oldFile)
meta.removeFile(protocol.GlobalDeviceID, oldFile)
}
// Add the new global to the global size counter
meta.addFile(globalDeviceID, newGlobal)
meta.addFile(protocol.GlobalDeviceID, newGlobal)
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
t.Put(gk, mustMarshal(&fl))
@ -197,7 +197,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
// didn't exist anyway, apparently
continue
}
meta.removeFile(globalDeviceID, f)
meta.removeFile(protocol.GlobalDeviceID, f)
removed = true
}
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
@ -215,7 +215,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
meta.addFile(globalDeviceID, f)
meta.addFile(protocol.GlobalDeviceID, f)
}
}
}

135
lib/db/meta.go
View File

@ -7,25 +7,30 @@
package db
import (
"bytes"
"math/bits"
"time"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
)
// like protocol.LocalDeviceID but with 0xf8 in all positions
var globalDeviceID = protocol.DeviceID{0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8}
// metadataTracker keeps metadata on a per device, per local flag basis.
type metadataTracker struct {
mut sync.RWMutex
counts CountsSet
indexes map[protocol.DeviceID]int // device ID -> index in counts
indexes map[metaKey]int // device ID + local flags -> index in counts
}
type metaKey struct {
dev protocol.DeviceID
flags uint32
}
func newMetadataTracker() *metadataTracker {
return &metadataTracker{
mut: sync.NewRWMutex(),
indexes: make(map[protocol.DeviceID]int),
indexes: make(map[metaKey]int),
}
}
@ -38,7 +43,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error {
// Initialize the index map
for i, c := range m.counts.Counts {
m.indexes[protocol.DeviceIDFromBytes(c.DeviceID)] = i
m.indexes[metaKey{protocol.DeviceIDFromBytes(c.DeviceID), c.LocalFlags}] = i
}
return nil
}
@ -72,14 +77,15 @@ func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
// countsPtr returns a pointer to the corresponding Counts struct, if
// necessary allocating one in the process
func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts {
func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flags uint32) *Counts {
// must be called with the mutex held
idx, ok := m.indexes[dev]
key := metaKey{dev, flags}
idx, ok := m.indexes[key]
if !ok {
idx = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:]})
m.indexes[dev] = idx
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flags})
m.indexes[key] = idx
}
return &m.counts.Counts[idx]
}
@ -87,12 +93,23 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts {
// addFile adds a file to the counts, adjusting the sequence number as
// appropriate
func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
if f.IsInvalid() {
return
m.mut.Lock()
if flags := f.FileLocalFlags(); flags == 0 {
// Account regular files in the zero-flags bucket.
m.addFileLocked(dev, 0, f)
} else {
// Account in flag specific buckets.
eachFlagBit(flags, func(flag uint32) {
m.addFileLocked(dev, flag, f)
})
}
m.mut.Lock()
cp := m.countsPtr(dev)
m.mut.Unlock()
}
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, flags)
switch {
case f.IsDeleted():
@ -109,18 +126,27 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
if seq := f.SequenceNo(); seq > cp.Sequence {
cp.Sequence = seq
}
m.mut.Unlock()
}
// removeFile removes a file from the counts
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
if f.IsInvalid() {
return
m.mut.Lock()
if flags := f.FileLocalFlags(); flags == 0 {
// Remove regular files from the zero-flags bucket
m.removeFileLocked(dev, 0, f)
} else {
// Remove from flag specific buckets.
eachFlagBit(flags, func(flag uint32) {
m.removeFileLocked(dev, flag, f)
})
}
m.mut.Lock()
cp := m.countsPtr(dev)
m.mut.Unlock()
}
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, f.FileLocalFlags())
switch {
case f.IsDeleted():
@ -153,14 +179,19 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
cp.Symlinks = 0
m.counts.Created = 0
}
m.mut.Unlock()
}
// resetAll resets all metadata for the given device
func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
m.mut.Lock()
*m.countsPtr(dev) = Counts{DeviceID: dev[:]}
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
LocalFlags: c.LocalFlags,
}
}
}
m.mut.Unlock()
}
@ -169,23 +200,30 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.mut.Lock()
c := m.countsPtr(dev)
c.Bytes = 0
c.Deleted = 0
c.Directories = 0
c.Files = 0
c.Symlinks = 0
// c.Sequence deliberately untouched
for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
Sequence: c.Sequence,
LocalFlags: c.LocalFlags,
}
}
}
m.mut.Unlock()
}
// Counts returns the counts for the given device ID
func (m *metadataTracker) Counts(dev protocol.DeviceID) Counts {
// Counts returns the counts for the given device ID and flag. `flag` should
// be zero or have exactly one bit set.
func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
m.mut.RLock()
defer m.mut.RUnlock()
idx, ok := m.indexes[dev]
idx, ok := m.indexes[metaKey{dev, flag}]
if !ok {
return Counts{}
}
@ -198,7 +236,7 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 {
m.mut.Lock()
defer m.mut.Unlock()
c := m.countsPtr(dev)
c := m.countsPtr(dev, 0)
c.Sequence++
return c.Sequence
}
@ -206,21 +244,26 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 {
// devices returns the list of devices tracked, excluding the local device
// (which we don't know the ID of)
func (m *metadataTracker) devices() []protocol.DeviceID {
devs := make([]protocol.DeviceID, 0, len(m.counts.Counts))
devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts))
m.mut.RLock()
for _, dev := range m.counts.Counts {
if dev.Sequence > 0 {
id := protocol.DeviceIDFromBytes(dev.DeviceID)
if id == globalDeviceID || id == protocol.LocalDeviceID {
if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID {
continue
}
devs = append(devs, id)
devs[id] = struct{}{}
}
}
m.mut.RUnlock()
return devs
devList := make([]protocol.DeviceID, 0, len(devs))
for dev := range devs {
devList = append(devList, dev)
}
return devList
}
func (m *metadataTracker) Created() time.Time {
@ -234,3 +277,19 @@ func (m *metadataTracker) SetCreated() {
m.counts.Created = time.Now().UnixNano()
m.mut.Unlock()
}
// eachFlagBit calls the function once for every bit that is set in flags
func eachFlagBit(flags uint32, fn func(flag uint32)) {
// Test each bit from the right, as long as there are bits left in the
// flag set. Clear any bits found and stop testing as soon as there are
// no more bits set.
currentBit := uint32(1 << 0)
for flags != 0 {
if flags&currentBit != 0 {
fn(currentBit)
flags &^= currentBit
}
currentBit <<= 1
}
}

82
lib/db/meta_test.go Normal file
View File

@ -0,0 +1,82 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"math/bits"
"sort"
"testing"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestEachFlagBit(t *testing.T) {
cases := []struct {
flags uint32
iterations int
}{
{0, 0},
{1<<0 | 1<<3, 2},
{1 << 0, 1},
{1 << 31, 1},
{1<<10 | 1<<20 | 1<<30, 3},
}
for _, tc := range cases {
var flags uint32
iterations := 0
eachFlagBit(tc.flags, func(f uint32) {
iterations++
flags |= f
if bits.OnesCount32(f) != 1 {
t.Error("expected exactly one bit to be set in every call")
}
})
if flags != tc.flags {
t.Errorf("expected 0x%x flags, got 0x%x", tc.flags, flags)
}
if iterations != tc.iterations {
t.Errorf("expected %d iterations, got %d", tc.iterations, iterations)
}
}
}
func TestMetaDevices(t *testing.T) {
d1 := protocol.DeviceID{1}
d2 := protocol.DeviceID{2}
meta := newMetadataTracker()
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 2, LocalFlags: 2})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1})
// There are five device/flags combos
if l := len(meta.counts.Counts); l < 5 {
t.Error("expected at least five buckets, not", l)
}
// There are only two non-local devices
devs := meta.devices()
if l := len(devs); l != 2 {
t.Fatal("expected two devices, not", l)
}
// Check that we got the two devices we expect
sort.Slice(devs, func(a, b int) bool {
return devs[a].Compare(devs[b]) == -1
})
if devs[0] != d1 {
t.Error("first device should be d1")
}
if devs[1] != d2 {
t.Error("second device should be d2")
}
}

18
lib/db/set.go
View File

@ -37,8 +37,12 @@ type FileSet struct {
type FileIntf interface {
FileSize() int64
FileName() string
FileLocalFlags() uint32
IsDeleted() bool
IsInvalid() bool
IsIgnored() bool
IsUnsupported() bool
MustRescan() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
@ -248,15 +252,23 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
}
func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.meta.Counts(device).Sequence
return s.meta.Counts(device, 0).Sequence
}
func (s *FileSet) LocalSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID)
local := s.meta.Counts(protocol.LocalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
return local.Add(recvOnlyChanged)
}
func (s *FileSet) ReceiveOnlyChangedSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
}
func (s *FileSet) GlobalSize() Counts {
return s.meta.Counts(globalDeviceID)
global := s.meta.Counts(protocol.GlobalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly)
return global.Add(recvOnlyChanged)
}
func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {

118
lib/db/set_test.go
View File

@ -906,7 +906,7 @@ func TestWithHaveSequence(t *testing.T) {
i := 2
s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], false, false) {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
}
i++
@ -917,7 +917,7 @@ func TestWithHaveSequence(t *testing.T) {
func TestIssue4925(t *testing.T) {
ldb := db.OpenMemory()
folder := "test)"
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{
@ -955,7 +955,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Error("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(remote0Have[0], false, false) {
} else if !need[0].IsEquivalent(remote0Have[0]) {
t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
}
@ -981,7 +981,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, remoteDevice0); len(need) != 1 {
t.Error("Expected 1 need for remote 0, got", need)
} else if !need[0].IsEquivalent(localHave[0], false, false) {
} else if !need[0].IsEquivalent(localHave[0]) {
t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
}
@ -1017,7 +1017,7 @@ func TestIssue5007(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], false, false) {
} else if !need[0].IsEquivalent(fs[0]) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@ -1052,7 +1052,7 @@ func TestNeedDeleted(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], false, false) {
} else if !need[0].IsEquivalent(fs[0]) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
@ -1065,6 +1065,110 @@ func TestNeedDeleted(t *testing.T) {
}
}
func TestReceiveOnlyAccounting(t *testing.T) {
ldb := db.OpenMemory()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := protocol.DeviceID{1}
remote := protocol.DeviceID{2}
// Three files that have been created by the remote device
version := protocol.Vector{Counters: []protocol.Counter{{ID: remote.Short(), Value: 1}}}
files := fileList{
protocol.FileInfo{Name: "f1", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f2", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f3", Size: 10, Sequence: 1, Version: version},
}
// We have synced them locally
replace(s, protocol.LocalDeviceID, files)
replace(s, remote, files)
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files initially, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
t.Fatal("expected 30 local bytes initially, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files initially, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
t.Fatal("expected 30 global bytes initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
t.Fatal("expected 0 receive only changed files initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes initially, not", n)
}
// Detected a local change in a receive only folder
changed := files[0]
changed.Version = changed.Version.Update(local.Short())
changed.Size = 100
changed.ModifiedBy = local.Short()
changed.LocalFlags = protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
// Check that we see the files
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files after local change, not", n)
}
if n := s.LocalSize().Bytes; n != 120 {
t.Fatal("expected 120 local bytes after local change, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files after local change, not", n)
}
if n := s.GlobalSize().Bytes; n != 120 {
t.Fatal("expected 120 global bytes after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 1 {
t.Fatal("expected 1 receive only changed file after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 {
t.Fatal("expected 100 receive only changed btyes after local change, not", n)
}
// Fake a revert. That's a two step process, first converting our
// changed file into a less preferred variant, then pulling down the old
// version.
changed.Version = protocol.Vector{}
changed.LocalFlags &^= protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{files[0]})
// Check that we see the files, same data as initially
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files after revert, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
t.Fatal("expected 30 local bytes after revert, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files after revert, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
t.Fatal("expected 30 global bytes after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
t.Fatal("expected 0 receive only changed files after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes after revert, not", n)
}
}
func TestNeedAfterUnignore(t *testing.T) {
ldb := db.OpenMemory()
@ -1090,7 +1194,7 @@ func TestNeedAfterUnignore(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need)
} else if !need[0].IsEquivalent(remote, false, false) {
} else if !need[0].IsEquivalent(remote) {
t.Fatalf("Got %v, expected %v", need[0], remote)
}
}

25
lib/db/structs.go
View File

@ -40,6 +40,10 @@ func (f FileInfoTruncated) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0
}
func (f FileInfoTruncated) IsUnsupported() bool {
return f.LocalFlags&protocol.FlagLocalUnsupported != 0
}
func (f FileInfoTruncated) IsIgnored() bool {
return f.LocalFlags&protocol.FlagLocalIgnored != 0
}
@ -48,6 +52,10 @@ func (f FileInfoTruncated) MustRescan() bool {
return f.LocalFlags&protocol.FlagLocalMustRescan != 0
}
func (f FileInfoTruncated) IsReceiveOnlyChanged() bool {
return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Type == protocol.FileInfoTypeDirectory
}
@ -86,6 +94,10 @@ func (f FileInfoTruncated) FileName() string {
return f.Name
}
func (f FileInfoTruncated) FileLocalFlags() uint32 {
return f.LocalFlags
}
func (f FileInfoTruncated) ModTime() time.Time {
return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
}
@ -110,3 +122,16 @@ func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protoco
LocalFlags: protocol.FlagLocalIgnored,
}
}
func (c Counts) Add(other Counts) Counts {
return Counts{
Files: c.Files + other.Files,
Directories: c.Directories + other.Directories,
Symlinks: c.Symlinks + other.Symlinks,
Deleted: c.Deleted + other.Deleted,
Bytes: c.Bytes + other.Bytes,
Sequence: c.Sequence + other.Sequence,
DeviceID: protocol.EmptyDeviceID[:],
LocalFlags: c.LocalFlags | other.LocalFlags,
}
}

116
lib/db/structs.pb.go
View File

@ -91,6 +91,7 @@ type Counts struct {
Bytes int64 `protobuf:"varint,5,opt,name=bytes,proto3" json:"bytes,omitempty"`
Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"`
DeviceID []byte `protobuf:"bytes,17,opt,name=deviceID,proto3" json:"deviceID,omitempty"`
LocalFlags uint32 `protobuf:"varint,18,opt,name=localFlags,proto3" json:"localFlags,omitempty"`
}
func (m *Counts) Reset() { *m = Counts{} }
@ -357,6 +358,13 @@ func (m *Counts) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintStructs(dAtA, i, uint64(len(m.DeviceID)))
i += copy(dAtA[i:], m.DeviceID)
}
if m.LocalFlags != 0 {
dAtA[i] = 0x90
i++
dAtA[i] = 0x1
i++
i = encodeVarintStructs(dAtA, i, uint64(m.LocalFlags))
}
return i, nil
}
@ -526,6 +534,9 @@ func (m *Counts) ProtoSize() (n int) {
if l > 0 {
n += 2 + l + sovStructs(uint64(l))
}
if m.LocalFlags != 0 {
n += 2 + sovStructs(uint64(m.LocalFlags))
}
return n
}
@ -1312,6 +1323,25 @@ func (m *Counts) Unmarshal(dAtA []byte) error {
m.DeviceID = []byte{}
}
iNdEx = postIndex
case 18:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)
}
m.LocalFlags = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.LocalFlags |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
@ -1541,47 +1571,47 @@ var (
func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) }
var fileDescriptorStructs = []byte{
// 663 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x40,
0x10, 0xb6, 0x62, 0xf9, 0x6f, 0x6c, 0xa7, 0xc9, 0x12, 0x82, 0x30, 0xd4, 0x16, 0x86, 0x82, 0x28,
0xd4, 0x6e, 0x13, 0x7a, 0x69, 0x6f, 0x6a, 0x08, 0x18, 0x4a, 0x5b, 0xd6, 0x21, 0xa7, 0x82, 0xd1,
0xcf, 0xda, 0x59, 0x22, 0x6b, 0x1d, 0xed, 0x3a, 0x41, 0x79, 0x92, 0x1e, 0xf3, 0x30, 0x3d, 0xe4,
0xd8, 0x73, 0x0f, 0x26, 0x75, 0x2f, 0x7d, 0x8c, 0xb2, 0xbb, 0x92, 0xa2, 0xf6, 0xd4, 0xde, 0xe6,
0x9b, 0x9f, 0x9d, 0x6f, 0x66, 0xbe, 0x85, 0x2e, 0x17, 0xc9, 0x3a, 0x10, 0x7c, 0xb4, 0x4a, 0x98,
0x60, 0x68, 0x27, 0xf4, 0x7b, 0x2f, 0x16, 0x54, 0x5c, 0xac, 0xfd, 0x51, 0xc0, 0x96, 0xe3, 0x05,
0x5b, 0xb0, 0xb1, 0x0a, 0xf9, 0xeb, 0xb9, 0x42, 0x0a, 0x28, 0x4b, 0x97, 0xf4, 0x5e, 0x97, 0xd2,
0x79, 0x1a, 0x07, 0xe2, 0x82, 0xc6, 0x8b, 0x92, 0x15, 0x51, 0x5f, 0xbf, 0x10, 0xb0, 0x68, 0xec,
0x93, 0x95, 0x2e, 0x1b, 0x5e, 0x41, 0xfb, 0x94, 0x46, 0xe4, 0x9c, 0x24, 0x9c, 0xb2, 0x18, 0xbd,
0x84, 0xc6, 0xb5, 0x36, 0x2d, 0xc3, 0x36, 0x9c, 0xf6, 0xd1, 0xde, 0x28, 0x2f, 0x1a, 0x9d, 0x93,
0x40, 0xb0, 0xc4, 0x35, 0xef, 0x37, 0x83, 0x0a, 0xce, 0xd3, 0xd0, 0x21, 0xd4, 0x43, 0x72, 0x4d,
0x03, 0x62, 0xed, 0xd8, 0x86, 0xd3, 0xc1, 0x19, 0x42, 0x16, 0x34, 0x68, 0x7c, 0xed, 0x45, 0x34,
0xb4, 0xaa, 0xb6, 0xe1, 0x34, 0x71, 0x0e, 0x87, 0xa7, 0xd0, 0xce, 0xda, 0xbd, 0xa7, 0x5c, 0xa0,
0x57, 0xd0, 0xcc, 0xde, 0xe2, 0x96, 0x61, 0x57, 0x9d, 0xf6, 0xd1, 0x93, 0x51, 0xe8, 0x8f, 0x4a,
0xac, 0xb2, 0x96, 0x45, 0xda, 0x1b, 0xf3, 0xcb, 0xdd, 0xa0, 0x32, 0x7c, 0x30, 0x61, 0x5f, 0x66,
0x4d, 0xe2, 0x39, 0x3b, 0x4b, 0xd6, 0x71, 0xe0, 0x09, 0x12, 0x22, 0x04, 0x66, 0xec, 0x2d, 0x89,
0xa2, 0xdf, 0xc2, 0xca, 0x46, 0xcf, 0xc1, 0x14, 0xe9, 0x4a, 0x33, 0xdc, 0x3d, 0x3a, 0x7c, 0x1c,
0xa9, 0x28, 0x4f, 0x57, 0x04, 0xab, 0x1c, 0x59, 0xcf, 0xe9, 0x2d, 0x51, 0xa4, 0xab, 0x58, 0xd9,
0xc8, 0x86, 0xf6, 0x8a, 0x24, 0x4b, 0xca, 0x35, 0x4b, 0xd3, 0x36, 0x9c, 0x2e, 0x2e, 0xbb, 0xd0,
0x53, 0x80, 0x25, 0x0b, 0xe9, 0x9c, 0x92, 0x70, 0xc6, 0xad, 0x9a, 0xaa, 0x6d, 0xe5, 0x9e, 0xa9,
0x5c, 0x46, 0x48, 0x22, 0x22, 0x48, 0x68, 0xd5, 0xf5, 0x32, 0x32, 0x88, 0x9c, 0xc7, 0x35, 0x35,
0x64, 0xc4, 0xdd, 0xdd, 0x6e, 0x06, 0x80, 0xbd, 0x9b, 0x89, 0xf6, 0x16, 0x6b, 0x43, 0xcf, 0x60,
0x37, 0x66, 0xb3, 0x32, 0x8f, 0xa6, 0x7a, 0xaa, 0x1b, 0xb3, 0x4f, 0x25, 0x26, 0xa5, 0x0b, 0xb6,
0xfe, 0xed, 0x82, 0x3d, 0x68, 0x72, 0x72, 0xb5, 0x26, 0x71, 0x40, 0x2c, 0x50, 0xcc, 0x0b, 0x8c,
0x06, 0xd0, 0x2e, 0xe6, 0x8a, 0xb9, 0xd5, 0xb6, 0x0d, 0xa7, 0x86, 0x8b, 0x51, 0x3f, 0x70, 0xf4,
0xb9, 0x94, 0xe0, 0xa7, 0x56, 0xc7, 0x36, 0x1c, 0xd3, 0x7d, 0x2b, 0x1b, 0x7c, 0xdf, 0x0c, 0x8e,
0xff, 0x43, 0x93, 0xa3, 0xe9, 0x05, 0x4b, 0xc4, 0xe4, 0xe4, 0xf1, 0x75, 0x37, 0x45, 0x63, 0x00,
0x3f, 0x62, 0xc1, 0xe5, 0x4c, 0x9d, 0xa4, 0x2b, 0xbb, 0xbb, 0x7b, 0xdb, 0xcd, 0xa0, 0x83, 0xbd,
0x1b, 0x57, 0x06, 0xa6, 0xf4, 0x96, 0xe0, 0x96, 0x9f, 0x9b, 0x72, 0x49, 0x3c, 0x5d, 0x46, 0x34,
0xbe, 0x9c, 0x09, 0x2f, 0x59, 0x10, 0x61, 0xed, 0x2b, 0x1d, 0x74, 0x33, 0xef, 0x99, 0x72, 0xca,
0x83, 0x46, 0x2c, 0xf0, 0xa2, 0xd9, 0x3c, 0xf2, 0x16, 0xdc, 0xfa, 0xd5, 0x50, 0x17, 0x05, 0xe5,
0x3b, 0x95, 0xae, 0x4c, 0x62, 0x5f, 0x0d, 0xa8, 0xbf, 0x63, 0xeb, 0x58, 0x70, 0x74, 0x00, 0xb5,
0x39, 0x8d, 0x08, 0x57, 0xc2, 0xaa, 0x61, 0x0d, 0xe4, 0x43, 0x21, 0x4d, 0xd4, 0x5a, 0x29, 0xe1,
0x4a, 0x60, 0x35, 0x5c, 0x76, 0xa9, 0xed, 0xea, 0xde, 0x5c, 0x69, 0xaa, 0x86, 0x0b, 0x5c, 0x96,
0x85, 0xa9, 0x42, 0x85, 0x2c, 0x0e, 0xa0, 0xe6, 0xa7, 0x82, 0xe4, 0x52, 0xd2, 0xe0, 0x8f, 0x4b,
0xd5, 0xff, 0xba, 0x54, 0x0f, 0x9a, 0xfa, 0xe7, 0x4d, 0x4e, 0xd4, 0xcc, 0x1d, 0x5c, 0xe0, 0xe1,
0x47, 0x68, 0xe9, 0x29, 0xa6, 0x44, 0x20, 0x07, 0xea, 0x81, 0x02, 0xd9, 0x6f, 0x03, 0xf9, 0xdb,
0x74, 0x38, 0x53, 0x46, 0x16, 0x97, 0xf4, 0x82, 0x84, 0xc8, 0x5f, 0xa5, 0x06, 0xab, 0xe2, 0x1c,
0xba, 0x07, 0xf7, 0x3f, 0xfa, 0x95, 0xfb, 0x6d, 0xdf, 0xf8, 0xb6, 0xed, 0x1b, 0x0f, 0xdb, 0x7e,
0xe5, 0xee, 0x67, 0xdf, 0xf0, 0xeb, 0xea, 0x96, 0xc7, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x9a,
0x4b, 0x16, 0x44, 0xcd, 0x04, 0x00, 0x00,
// 671 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6b, 0xdb, 0x4c,
0x10, 0xb6, 0x62, 0xf9, 0x6b, 0x6c, 0xe7, 0x4d, 0x96, 0x10, 0x84, 0xe1, 0xb5, 0x85, 0xa1, 0x20,
0x0a, 0xb5, 0xdb, 0x84, 0x5e, 0xda, 0x9b, 0x1a, 0x02, 0x86, 0xd2, 0x96, 0x75, 0xc8, 0xa9, 0x60,
0xf4, 0xb1, 0x76, 0x96, 0xc8, 0x5a, 0x47, 0xbb, 0x4e, 0x50, 0x7e, 0x49, 0x8f, 0xf9, 0x39, 0x39,
0xf6, 0xdc, 0x83, 0x49, 0xdd, 0x1e, 0xfa, 0x33, 0xca, 0xee, 0x4a, 0x8a, 0x9a, 0x53, 0x7b, 0x9b,
0x67, 0x3e, 0x76, 0x9e, 0x99, 0x79, 0x16, 0xba, 0x5c, 0x24, 0xeb, 0x40, 0xf0, 0xd1, 0x2a, 0x61,
0x82, 0xa1, 0x9d, 0xd0, 0xef, 0xbd, 0x58, 0x50, 0x71, 0xb1, 0xf6, 0x47, 0x01, 0x5b, 0x8e, 0x17,
0x6c, 0xc1, 0xc6, 0x2a, 0xe4, 0xaf, 0xe7, 0x0a, 0x29, 0xa0, 0x2c, 0x5d, 0xd2, 0x7b, 0x5d, 0x4a,
0xe7, 0x69, 0x1c, 0x88, 0x0b, 0x1a, 0x2f, 0x4a, 0x56, 0x44, 0x7d, 0xfd, 0x42, 0xc0, 0xa2, 0xb1,
0x4f, 0x56, 0xba, 0x6c, 0x78, 0x05, 0xed, 0x53, 0x1a, 0x91, 0x73, 0x92, 0x70, 0xca, 0x62, 0xf4,
0x12, 0x1a, 0xd7, 0xda, 0xb4, 0x0c, 0xdb, 0x70, 0xda, 0x47, 0x7b, 0xa3, 0xbc, 0x68, 0x74, 0x4e,
0x02, 0xc1, 0x12, 0xd7, 0xbc, 0xdf, 0x0c, 0x2a, 0x38, 0x4f, 0x43, 0x87, 0x50, 0x0f, 0xc9, 0x35,
0x0d, 0x88, 0xb5, 0x63, 0x1b, 0x4e, 0x07, 0x67, 0x08, 0x59, 0xd0, 0xa0, 0xf1, 0xb5, 0x17, 0xd1,
0xd0, 0xaa, 0xda, 0x86, 0xd3, 0xc4, 0x39, 0x1c, 0x9e, 0x42, 0x3b, 0x6b, 0xf7, 0x9e, 0x72, 0x81,
0x5e, 0x41, 0x33, 0x7b, 0x8b, 0x5b, 0x86, 0x5d, 0x75, 0xda, 0x47, 0xff, 0x8d, 0x42, 0x7f, 0x54,
0x62, 0x95, 0xb5, 0x2c, 0xd2, 0xde, 0x98, 0x5f, 0xee, 0x06, 0x95, 0xe1, 0x83, 0x09, 0xfb, 0x32,
0x6b, 0x12, 0xcf, 0xd9, 0x59, 0xb2, 0x8e, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, 0x6f, 0x49,
0x14, 0xfd, 0x16, 0x56, 0x36, 0x7a, 0x0e, 0xa6, 0x48, 0x57, 0x9a, 0xe1, 0xee, 0xd1, 0xe1, 0xe3,
0x48, 0x45, 0x79, 0xba, 0x22, 0x58, 0xe5, 0xc8, 0x7a, 0x4e, 0x6f, 0x89, 0x22, 0x5d, 0xc5, 0xca,
0x46, 0x36, 0xb4, 0x57, 0x24, 0x59, 0x52, 0xae, 0x59, 0x9a, 0xb6, 0xe1, 0x74, 0x71, 0xd9, 0x85,
0xfe, 0x07, 0x58, 0xb2, 0x90, 0xce, 0x29, 0x09, 0x67, 0xdc, 0xaa, 0xa9, 0xda, 0x56, 0xee, 0x99,
0xca, 0x65, 0x84, 0x24, 0x22, 0x82, 0x84, 0x56, 0x5d, 0x2f, 0x23, 0x83, 0xc8, 0x79, 0x5c, 0x53,
0x43, 0x46, 0xdc, 0xdd, 0xed, 0x66, 0x00, 0xd8, 0xbb, 0x99, 0x68, 0x6f, 0xb1, 0x36, 0xf4, 0x0c,
0x76, 0x63, 0x36, 0x2b, 0xf3, 0x68, 0xaa, 0xa7, 0xba, 0x31, 0xfb, 0x54, 0x62, 0x52, 0xba, 0x60,
0xeb, 0xef, 0x2e, 0xd8, 0x83, 0x26, 0x27, 0x57, 0x6b, 0x12, 0x07, 0xc4, 0x02, 0xc5, 0xbc, 0xc0,
0x68, 0x00, 0xed, 0x62, 0xae, 0x98, 0x5b, 0x6d, 0xdb, 0x70, 0x6a, 0xb8, 0x18, 0xf5, 0x03, 0x47,
0x9f, 0x4b, 0x09, 0x7e, 0x6a, 0x75, 0x6c, 0xc3, 0x31, 0xdd, 0xb7, 0xb2, 0xc1, 0xb7, 0xcd, 0xe0,
0xf8, 0x1f, 0x34, 0x39, 0x9a, 0x5e, 0xb0, 0x44, 0x4c, 0x4e, 0x1e, 0x5f, 0x77, 0x53, 0x34, 0x06,
0xf0, 0x23, 0x16, 0x5c, 0xce, 0xd4, 0x49, 0xba, 0xb2, 0xbb, 0xbb, 0xb7, 0xdd, 0x0c, 0x3a, 0xd8,
0xbb, 0x71, 0x65, 0x60, 0x4a, 0x6f, 0x09, 0x6e, 0xf9, 0xb9, 0x29, 0x97, 0xc4, 0xd3, 0x65, 0x44,
0xe3, 0xcb, 0x99, 0xf0, 0x92, 0x05, 0x11, 0xd6, 0xbe, 0xd2, 0x41, 0x37, 0xf3, 0x9e, 0x29, 0xa7,
0x3c, 0x68, 0xc4, 0x02, 0x2f, 0x9a, 0xcd, 0x23, 0x6f, 0xc1, 0xad, 0x5f, 0x0d, 0x75, 0x51, 0x50,
0xbe, 0x53, 0xe9, 0xca, 0x24, 0xf6, 0xd3, 0x80, 0xfa, 0x3b, 0xb6, 0x8e, 0x05, 0x47, 0x07, 0x50,
0x9b, 0xd3, 0x88, 0x70, 0x25, 0xac, 0x1a, 0xd6, 0x40, 0x3e, 0x14, 0xd2, 0x44, 0xad, 0x95, 0x12,
0xae, 0x04, 0x56, 0xc3, 0x65, 0x97, 0xda, 0xae, 0xee, 0xcd, 0x95, 0xa6, 0x6a, 0xb8, 0xc0, 0x65,
0x59, 0x98, 0x2a, 0x54, 0xc8, 0xe2, 0x00, 0x6a, 0x7e, 0x2a, 0x48, 0x2e, 0x25, 0x0d, 0xfe, 0xb8,
0x54, 0xfd, 0xc9, 0xa5, 0x7a, 0xd0, 0xd4, 0x3f, 0x6f, 0x72, 0xa2, 0x66, 0xee, 0xe0, 0x02, 0xa3,
0x3e, 0x94, 0x46, 0xb3, 0xd0, 0xd3, 0x61, 0x87, 0x1f, 0xa1, 0xa5, 0xa7, 0x9c, 0x12, 0x81, 0x1c,
0xa8, 0x07, 0x0a, 0x64, 0xbf, 0x11, 0xe4, 0x6f, 0xd4, 0xe1, 0x4c, 0x39, 0x59, 0x5c, 0xd2, 0x0f,
0x12, 0x22, 0x7f, 0x9d, 0x1a, 0xbc, 0x8a, 0x73, 0xe8, 0x1e, 0xdc, 0x7f, 0xef, 0x57, 0xee, 0xb7,
0x7d, 0xe3, 0xeb, 0xb6, 0x6f, 0x3c, 0x6c, 0xfb, 0x95, 0xbb, 0x1f, 0x7d, 0xc3, 0xaf, 0xab, 0x5b,
0x1f, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x4d, 0xd7, 0x14, 0xed, 0x04, 0x00, 0x00,
}

15
lib/db/structs.proto
View File

@ -46,13 +46,14 @@ message FileInfoTruncated {
// For each folder and device we keep one of these to track the current
// counts and sequence. We also keep one for the global state of the folder.
message Counts {
int32 files = 1;
int32 directories = 2;
int32 symlinks = 3;
int32 deleted = 4;
int64 bytes = 5;
int64 sequence = 6; // zero for the global state
bytes deviceID = 17; // device ID for remote devices, or special values for local/global
int32 files = 1;
int32 directories = 2;
int32 symlinks = 3;
int32 deleted = 4;
int64 bytes = 5;
int64 sequence = 6; // zero for the global state
bytes deviceID = 17; // device ID for remote devices, or special values for local/global
uint32 localFlags = 18; // the local flag for this count bucket
}
message CountsSet {

5
lib/model/folder.go
View File

@ -27,6 +27,7 @@ var errWatchNotStarted = errors.New("not started")
type folder struct {
stateTracker
config.FolderConfiguration
localFlags uint32
model *Model
shortID protocol.ShortID
@ -175,6 +176,8 @@ func (f *folder) BringToFront(string) {}
func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) DelayScan(next time.Duration) {
f.Delay(next)
}
@ -263,7 +266,7 @@ func (f *folder) getHealthError() error {
}
func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs, f.localFlags); err != nil {
// Potentially sets the error twice, once in the scanner just
// by doing a check, and once here, if the error returned is
// the same one as returned by CheckHealth, though

210
lib/model/folder_recvonly.go Normal file
View File

@ -0,0 +1,210 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"sort"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/versioner"
)
func init() {
folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder
}
/*
receiveOnlyFolder is a folder that does not propagate local changes outward.
It does this by the following general mechanism (not all of which is
implemted in this file):
- Local changes are scanned and versioned as usual, but get the
FlagLocalReceiveOnly bit set.
- When changes are sent to the cluster this bit gets converted to the
Invalid bit (like all other local flags, currently) and also the Version
gets set to the empty version. The reason for clearing the Version is to
ensure that other devices will not consider themselves out of date due to
our change.
- The database layer accounts sizes per flag bit, so we can know how many
files have been changed locally. We use this to trigger a "Revert" option
on the folder when the amount of locally changed data is nonzero.
- To revert we take the files which have changed and reset their version
counter down to zero. The next pull will replace our changed version with
the globally latest. As this is a user-initiated operation we do not cause
conflict copies when reverting.
- When pulling normally (i.e., not in the revert case) with local changes,
normal conflict resolution will apply. Conflict copies will be created,
but not propagated outwards (because receive only, right).
Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that
sets an extra bit on local changes and has a Revert method.
*/
type receiveOnlyFolder struct {
*sendReceiveFolder
}
func newReceiveOnlyFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
sr := newSendReceiveFolder(model, cfg, ver, fs).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr}
}
func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
// XXX: This *really* should be given to us in the constructor...
f.model.fmut.RLock()
ignores := f.model.folderIgnores[f.folderID]
f.model.fmut.RUnlock()
delQueue := &deleteQueue{
handler: f, // for the deleteFile and deleteDir methods
ignores: ignores,
}
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
// receive only mode.
return true
}
if len(fi.Version.Counters) == 1 && fi.Version.Counters[0].ID == f.shortID {
// We are the only device mentioned in the version vector so the
// file must originate here. A revert then means to delete it.
// We'll delete files directly, directories get queued and
// handled below.
handled, err := delQueue.handle(fi)
if err != nil {
l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
return true // continue
}
if !handled {
return true // continue
}
fi = protocol.FileInfo{
Name: fi.Name,
Type: fi.Type,
ModifiedS: fi.ModifiedS,
ModifiedNs: fi.ModifiedNs,
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{}, // if this file ever resurfaces anywhere we want our delete to be strictly older
}
} else {
// Revert means to throw away our local changes. We reset the
// version to the empty vector, which is strictly older than any
// other existing version. It is not in conflict with anything,
// either, so we will not create a conflict copy of our local
// changes.
fi.Version = protocol.Vector{}
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
batch = append(batch, fi)
batchSizeBytes += fi.ProtoSize()
if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes {
updateFn(batch)
batch = batch[:0]
batchSizeBytes = 0
}
return true
})
if len(batch) > 0 {
updateFn(batch)
}
batch = batch[:0]
batchSizeBytes = 0
// Handle any queued directories
deleted, err := delQueue.flush()
if err != nil {
l.Infoln("Revert:", err)
}
now := time.Now()
for _, dir := range deleted {
batch = append(batch, protocol.FileInfo{
Name: dir,
Type: protocol.FileInfoTypeDirectory,
ModifiedS: now.Unix(),
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{},
})
}
if len(batch) > 0 {
updateFn(batch)
}
// We will likely have changed our local index, but that won't trigger a
// pull by itself. Make sure we schedule one so that we start
// downloading files.
f.SchedulePull()
}
// deleteQueue handles deletes by delegating to a handler and queuing
// directories for last.
type deleteQueue struct {
handler interface {
deleteFile(file protocol.FileInfo) (dbUpdateJob, error)
deleteDir(dir string, ignores *ignore.Matcher, scanChan chan<- string) error
}
ignores *ignore.Matcher
dirs []string
}
func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
// Things that are ignored but not marked deletable are not processed.
ign := q.ignores.Match(fi.Name)
if ign.IsIgnored() && !ign.IsDeletable() {
return false, nil
}
// Directories are queued for later processing.
if fi.IsDirectory() {
q.dirs = append(q.dirs, fi.Name)
return false, nil
}
// Kill it.
_, err := q.handler.deleteFile(fi)
return true, err
}
func (q *deleteQueue) flush() ([]string, error) {
// Process directories from the leaves inward.
sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
var firstError error
var deleted []string
for _, dir := range q.dirs {
if err := q.handler.deleteDir(dir, q.ignores, nil); err == nil {
deleted = append(deleted, dir)
} else if err != nil && firstError == nil {
firstError = err
}
}
return deleted, firstError
}

261
lib/model/folder_recvonly_test.go Normal file
View File

@ -0,0 +1,261 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestRecvOnlyRevertDeletes(t *testing.T) {
// Make sure that we delete extraneous files and directories when we hit
// Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
os.MkdirAll("_recvonly/ignDir", 0755)
os.MkdirAll("_recvonly/unknownDir", 0755)
ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644)
knownFiles := setupKnownFiles(t, []byte("hello\n"))
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
// Start the folder. This will cause a scan, should discover the other stuff in the folder
m.StartFolder("ro")
m.ScanFolder("ro")
// We should now have two files and two directories.
size = m.GlobalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// Revert should delete the unknown stuff
m.Revert("ro")
// These should still exist
for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} {
_, err := os.Stat(p)
if err != nil {
t.Error("Unexpected error:", err)
}
}
// These should have been removed
for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} {
_, err := os.Stat(p)
if !os.IsNotExist(err) {
t.Error("Unexpected existing thing:", p)
}
}
// We should now have one file and directory again.
size = m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
}
}
func TestRecvOnlyRevertNeeds(t *testing.T) {
// Make sure that a new file gets picked up and considered latest, then
// gets considered old when we hit Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
oldData := []byte("hello\n")
knownFiles := setupKnownFiles(t, oldData)
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
// Start the folder. This will cause a scan.
m.StartFolder("ro")
m.ScanFolder("ro")
// Everything should be in sync.
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
// Update the file.
newData := []byte("totally different data\n")
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644); err != nil {
t.Fatal(err)
}
// Rescan.
if err := m.ScanFolder("ro"); err != nil {
t.Fatal(err)
}
// We now have a newer file than the rest of the cluster. Global state should reflect this.
size = m.GlobalSize("ro")
const sizeOfDir = 128
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Global: expected the new file to be reflected: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// We hit the Revert button. The file that was new should become old.
m.Revert("ro")
size = m.GlobalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected the global size to revert: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the local size to remain: %+v", size)
}
size = m.NeedSize("ro")
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
t.Fatalf("Local: expected to need the old file data: %+v", size)
}
}
func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo {
if err := os.MkdirAll("_recvonly/knownDir", 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644); err != nil {
t.Fatal(err)
}
t0 := time.Now().Add(-1 * time.Minute)
if err := os.Chtimes("_recvonly/knownDir/knownFile", t0, t0); err != nil {
t.Fatal(err)
}
fi, err := os.Stat("_recvonly/knownDir/knownFile")
if err != nil {
t.Fatal(err)
}
knownFiles := []protocol.FileInfo{
{
Name: "knownDir",
Type: protocol.FileInfoTypeDirectory,
Permissions: 0755,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
{
Name: "knownDir/knownFile",
Type: protocol.FileInfoTypeFile,
Permissions: 0644,
Size: fi.Size(),
ModifiedS: fi.ModTime().Unix(),
ModifiedNs: int32(fi.ModTime().UnixNano() % 1e9),
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
}
return knownFiles
}
func setupROFolder() *Model {
fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly")
fcfg.Type = config.FolderTypeReceiveOnly
fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
cfg := defaultCfg.Copy()
cfg.Folders = append(cfg.Folders, fcfg)
wrp := config.Wrap("/dev/null", cfg)
db := db.OpenMemory()
m := NewModel(wrp, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
m.ServeBackground()
m.AddFolder(fcfg)
return m
}

2
lib/model/folder_sendonly.go
View File

@ -76,7 +76,7 @@ func (f *sendOnlyFolder) pull() bool {
}
file := intf.(protocol.FileInfo)
if !file.IsEquivalent(curFile, f.IgnorePerms, false) {
if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) {
return true
}

32
lib/model/folder_sendrecv.go
View File

@ -501,7 +501,11 @@ func (f *sendReceiveFolder) processDeletions(ignores *ignore.Matcher, fileDeleti
}
l.Debugln(f, "Deleting file", file.Name)
f.deleteFile(file, dbUpdateChan)
if update, err := f.deleteFile(file); err != nil {
f.newError("delete file", file.Name, err)
} else {
dbUpdateChan <- update
}
}
for i := range dirDeletions {
@ -736,7 +740,7 @@ func (f *sendReceiveFolder) handleDeleteDir(file protocol.FileInfo, ignores *ign
}
// deleteFile attempts to delete the given file
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) {
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) (dbUpdateJob, error) {
// Used in the defer closure below, updated by the function body. Take
// care not declare another err.
var err error
@ -775,16 +779,18 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan
if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
return dbUpdateJob{file, dbUpdateDeleteFile}, nil
}
if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the file, and it's not a permission
// problem. Lets assume the error is in fact some variant of "file
// does not exist" (possibly expressed as some parent being a file and
// not a directory etc) and that the delete is handled.
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
} else {
f.newError("delete file", file.Name, err)
return dbUpdateJob{file, dbUpdateDeleteFile}, nil
}
return dbUpdateJob{}, err
}
// renameFile attempts to rename an existing file to a destination
@ -1778,10 +1784,14 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC
} else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() {
hasIgnored = true
} else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
// Something appeared in the dir that we either are not
// aware of at all, that we think should be deleted or that
// is invalid, but not currently ignored -> schedule scan
scanChan <- fullDirFile
// Something appeared in the dir that we either are not aware of
// at all, that we think should be deleted or that is invalid,
// but not currently ignored -> schedule scan. The scanChan
// might be nil, in which case we trust the scanning to be
// handled later as a result of our error return.
if scanChan != nil {
scanChan <- fullDirFile
}
hasToBeScanned = true
} else {
// Dir contains file that is valid according to db and

4
lib/model/folder_sendrecv_test.go
View File

@ -74,12 +74,12 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
}
}
func setUpModel(file protocol.FileInfo) *Model {
func setUpModel(files ...protocol.FileInfo) *Model {
db := db.OpenMemory()
model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
model.AddFolder(defaultFolderConfig)
// Update index
model.updateLocalsFromScanning("default", []protocol.FileInfo{file})
model.updateLocalsFromScanning("default", files)
return model
}

41
lib/model/model.go
View File

@ -57,6 +57,7 @@ const (
type service interface {
BringToFront(string)
Override(*db.FileSet, func([]protocol.FileInfo))
Revert(*db.FileSet, func([]protocol.FileInfo))
DelayScan(d time.Duration)
IgnoresUpdated() // ignore matcher was updated notification
SchedulePull() // something relevant changed, we should try a pull
@ -690,6 +691,18 @@ func (m *Model) LocalSize(folder string) db.Counts {
return db.Counts{}
}
// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *Model) ReceiveOnlyChangedSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
return rf.ReceiveOnlyChangedSize()
}
return db.Counts{}
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) db.Counts {
m.fmut.RLock()
@ -1747,6 +1760,12 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
// Mark the file as invalid if any of the local bad stuff flags are set.
f.RawInvalid = f.IsInvalid()
// If the file is marked LocalReceive (i.e., changed locally on a
// receive only folder) we do not want it to ever become the
// globally best version, invalid or not.
if f.IsReceiveOnlyChanged() {
f.Version = protocol.Vector{}
}
f.LocalFlags = 0 // never sent externally
if dropSymlinks && f.IsSymlink() {
@ -1940,7 +1959,7 @@ func (m *Model) ScanFolderSubdirs(folder string, subs []string) error {
return runner.Scan(subs)
}
func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string) error {
func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string, localFlags uint32) error {
m.fmut.RLock()
if err := m.checkFolderRunningLocked(folder); err != nil {
m.fmut.RUnlock()
@ -2010,6 +2029,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseLargeBlocks: folderCfg.UseLargeBlocks,
LocalFlags: localFlags,
})
if err := runner.CheckHealth(); err != nil {
@ -2106,6 +2126,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ModifiedBy: m.id.Short(),
Deleted: true,
Version: f.Version.Update(m.shortID),
LocalFlags: localFlags,
}
// We do not want to override the global version
// with the deleted file. Keeping only our local
@ -2289,6 +2310,24 @@ func (m *Model) Override(folder string) {
})
}
func (m *Model) Revert(folder string) {
// Grab the runner and the file set.
m.fmut.RLock()
fs, fsOK := m.folderFiles[folder]
runner, runnerOK := m.folderRunners[folder]
m.fmut.RUnlock()
if !fsOK || !runnerOK {
return
}
// Run the revert, taking updates as if they came from scanning.
runner.Revert(fs, func(files []protocol.FileInfo) {
m.updateLocalsFromScanning(folder, files)
})
}
// CurrentSequence returns the change version for the given folder.
// This is guaranteed to increment if the contents of the local folder has
// changed.

30
lib/protocol/bep_extensions.go
View File

@ -49,6 +49,10 @@ func (f FileInfo) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0
}
func (f FileInfo) IsUnsupported() bool {
return f.LocalFlags&FlagLocalUnsupported != 0
}
func (f FileInfo) IsIgnored() bool {
return f.LocalFlags&FlagLocalIgnored != 0
}
@ -57,6 +61,10 @@ func (f FileInfo) MustRescan() bool {
return f.LocalFlags&FlagLocalMustRescan != 0
}
func (f FileInfo) IsReceiveOnlyChanged() bool {
return f.LocalFlags&FlagLocalReceiveOnly != 0
}
func (f FileInfo) IsDirectory() bool {
return f.Type == FileInfoTypeDirectory
}
@ -99,6 +107,10 @@ func (f FileInfo) FileName() string {
return f.Name
}
func (f FileInfo) FileLocalFlags() uint32 {
return f.LocalFlags
}
func (f FileInfo) ModTime() time.Time {
return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
}
@ -114,7 +126,7 @@ func (f FileInfo) FileVersion() Vector {
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {
// If only one of the files is invalid, that one loses
// If only one of the files is invalid, that one loses.
if f.IsInvalid() != other.IsInvalid() {
return !f.IsInvalid()
}
@ -145,7 +157,15 @@ func (f FileInfo) IsEmpty() bool {
return f.Version.Counters == nil
}
// IsEquivalent checks that the two file infos represent the same actual file content,
func (f FileInfo) IsEquivalent(other FileInfo) bool {
return f.isEquivalent(other, false, false, 0)
}
func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags)
}
// isEquivalent checks that the two file infos represent the same actual file content,
// i.e. it does purposely not check only selected (see below) struct members.
// Permissions (config) and blocks (scanning) can be excluded from the comparison.
// Any file info is not "equivalent", if it has different
@ -160,7 +180,7 @@ func (f FileInfo) IsEmpty() bool {
// A symlink is not "equivalent", if it has different
// - target
// A directory does not have anything specific to check.
func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool) bool {
func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
if f.MustRescan() || other.MustRescan() {
// These are per definition not equivalent because they don't
// represent a valid state, even if both happen to have the
@ -168,6 +188,10 @@ func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
return false
}
// Mask out the ignored local flags before checking IsInvalid() below
f.LocalFlags &^= ignoreFlags
other.LocalFlags &^= ignoreFlags
if f.Name != other.Name || f.Type != other.Type || f.Deleted != other.Deleted || f.IsInvalid() != other.IsInvalid() {
return false
}

12
lib/protocol/deviceid.go
View File

@ -19,10 +19,18 @@ type DeviceID [DeviceIDLength]byte
type ShortID uint64
var (
LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
EmptyDeviceID = DeviceID{ /* all zeroes */ }
LocalDeviceID = repeatedDeviceID(0xff)
GlobalDeviceID = repeatedDeviceID(0xf8)
EmptyDeviceID = DeviceID{ /* all zeroes */ }
)
func repeatedDeviceID(v byte) (d DeviceID) {
for i := range d {
d[i] = v
}
return
}
// NewDeviceID generates a new device ID from the raw bytes of a certificate
func NewDeviceID(rawCert []byte) DeviceID {
var n DeviceID

5
lib/protocol/protocol.go
View File

@ -94,14 +94,15 @@ const (
FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // Change detected on receive only folder
// Flags that should result in the Invalid bit on outgoing updates
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly
// Flags that should result in a file being in conflict with its
// successor, due to us not having an up to date picture of its state on
// disk.
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly
)
var (

16
lib/protocol/protocol_test.go
View File

@ -423,6 +423,7 @@ func TestIsEquivalent(t *testing.T) {
b FileInfo
ignPerms *bool // nil means should not matter, we'll test both variants
ignBlocks *bool
ignFlags uint32
eq bool
}
cases := []testCase{
@ -491,6 +492,17 @@ func TestIsEquivalent(t *testing.T) {
b: FileInfo{LocalFlags: FlagLocalUnsupported},
eq: true,
},
{
a: FileInfo{LocalFlags: 0},
b: FileInfo{LocalFlags: FlagLocalReceiveOnly},
eq: false,
},
{
a: FileInfo{LocalFlags: 0},
b: FileInfo{LocalFlags: FlagLocalReceiveOnly},
ignFlags: FlagLocalReceiveOnly,
eq: true,
},
// Difference in blocks is not OK
{
@ -588,10 +600,10 @@ func TestIsEquivalent(t *testing.T) {
continue
}
if res := tc.a.IsEquivalent(tc.b, ignPerms, ignBlocks); res != tc.eq {
if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
if res := tc.b.IsEquivalent(tc.a, ignPerms, ignBlocks); res != tc.eq {
if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
}
}

11
lib/scanner/walk.go
View File

@ -68,6 +68,8 @@ type Config struct {
ProgressTickIntervalS int
// Whether to use large blocks for large files or the old standard of 128KiB for everything.
UseLargeBlocks bool
// Local flags to set on scanned files
LocalFlags uint32
}
type CurrentFiler interface {
@ -367,10 +369,11 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
ModifiedBy: w.ShortID,
Size: info.Size(),
RawBlockSize: int32(blockSize),
LocalFlags: w.LocalFlags,
}
if hasCurFile {
if curFile.IsEquivalent(f, w.IgnorePerms, true) {
if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if curFile.ShouldConflict() {
@ -407,10 +410,11 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
ModifiedS: info.ModTime().Unix(),
ModifiedNs: int32(info.ModTime().Nanosecond()),
ModifiedBy: w.ShortID,
LocalFlags: w.LocalFlags,
}
if ok {
if cf.IsEquivalent(f, w.IgnorePerms, true) {
if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if cf.ShouldConflict() {
@ -463,10 +467,11 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan pro
NoPermissions: true, // Symlinks don't have permissions of their own
SymlinkTarget: target,
ModifiedBy: w.ShortID,
LocalFlags: w.LocalFlags,
}
if ok {
if cf.IsEquivalent(f, w.IgnorePerms, true) {
if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil
}
if cf.ShouldConflict() {

69
lib/scanner/walk_test.go
View File

@ -221,8 +221,8 @@ func TestNormalization(t *testing.T) {
// make sure it all gets done. In production, things will be correct
// eventually...
walkDir(testFs, "normalization", nil, nil)
tmp := walkDir(testFs, "normalization", nil, nil)
walkDir(testFs, "normalization", nil, nil, 0)
tmp := walkDir(testFs, "normalization", nil, nil, 0)
files := fileList(tmp).testfiles()
@ -267,7 +267,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil)
files := walkDir(fs, path, nil, nil, 0)
// Verify that we got one symlink and with the correct attributes
if len(files) != 1 {
@ -300,7 +300,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
for _, path := range []string{".", "link"} {
// Scan it
files := walkDir(fs, path, nil, nil)
files := walkDir(fs, path, nil, nil, 0)
// Verify that we got zero symlinks
if len(files) != 0 {
@ -329,7 +329,7 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan it
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil)
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
// Verify that we got two files
if len(files) != 2 {
@ -353,7 +353,7 @@ func TestBlocksizeHysteresis(t *testing.T) {
current := make(fakeCurrentFiler)
runTest := func(expectedBlockSize int) {
files := walkDir(sf, ".", current, nil)
files := walkDir(sf, ".", current, nil, 0)
if len(files) != 1 {
t.Fatalf("expected one file, not %d", len(files))
}
@ -407,7 +407,57 @@ func TestBlocksizeHysteresis(t *testing.T) {
runTest(512 << 10)
}
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher) []protocol.FileInfo {
func TestWalkReceiveOnly(t *testing.T) {
sf := fs.NewWalkFilesystem(&singleFileFS{
name: "testfile.dat",
filesize: 1024,
})
current := make(fakeCurrentFiler)
// Initial scan, no files in the CurrentFiler. Should pick up the file and
// set the ReceiveOnly flag on it, because that's the flag we give the
// walker to set.
files := walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 1 {
t.Fatal("Should have scanned one file")
}
if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
t.Fatal("Should have set the ReceiveOnly flag")
}
// Update the CurrentFiler and scan again. It should not return
// anything, because the file has not changed. This verifies that the
// ReceiveOnly flag is properly ignored and doesn't trigger a rescan
// every time.
cur := files[0]
current[cur.Name] = cur
files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 0 {
t.Fatal("Should not have scanned anything")
}
// Now pretend the file was previously ignored instead. We should pick up
// the difference in flags and set just the LocalReceive flags.
cur.LocalFlags = protocol.FlagLocalIgnored
current[cur.Name] = cur
files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 1 {
t.Fatal("Should have scanned one file")
}
if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
t.Fatal("Should have set the ReceiveOnly flag")
}
}
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo {
fchan := Walk(context.TODO(), Config{
Filesystem: fs,
Subs: []string{dir},
@ -416,6 +466,7 @@ func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.
UseLargeBlocks: true,
CurrentFiler: cfiler,
Matcher: matcher,
LocalFlags: localFlags,
})
var tmp []protocol.FileInfo
@ -579,7 +630,7 @@ func TestIssue4799(t *testing.T) {
}
fd.Close()
files := walkDir(fs, "/foo", nil, nil)
files := walkDir(fs, "/foo", nil, nil, 0)
if len(files) != 1 || files[0].Name != "foo" {
t.Error(`Received unexpected file infos when walking "/foo"`, files)
}
@ -597,7 +648,7 @@ func TestRecurseInclude(t *testing.T) {
t.Fatal(err)
}
files := walkDir(testFs, ".", nil, ignores)
files := walkDir(testFs, ".", nil, ignores, 0)
expected := []string{
filepath.Join("dir1"),