Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Add receive only folder type (#5027) 

Adds a receive only folder type that does not send changes, and where the user can optionally revert local changes. Also changes some of the icons to make the three folder types distinguishable.
This commit is contained in:
Jakob Borg 2018-07-12 11:15:57 +03:00 committed by GitHub
parent 1a6c7587c2
commit f822b10550
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1136 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/syncthing/gui.go
View File

@ -85,6 +85,7 @@ type modelIntf interface {
GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
Completion(device protocol.DeviceID, folder string) model.FolderCompletion Completion(device protocol.DeviceID, folder string) model.FolderCompletion
Override(folder string) Override(folder string)
Revert(folder string)
NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
NeedSize(folder string) db.Counts NeedSize(folder string) db.Counts
@ -107,6 +108,7 @@ type modelIntf interface {
Connection(deviceID protocol.DeviceID) (connections.Connection, bool) Connection(deviceID protocol.DeviceID) (connections.Connection, bool)
GlobalSize(folder string) db.Counts GlobalSize(folder string) db.Counts
LocalSize(folder string) db.Counts LocalSize(folder string) db.Counts
ReceiveOnlyChangedSize(folder string) db.Counts
CurrentSequence(folder string) (int64, bool) CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool)
State(folder string) (string, time.Time, error) State(folder string) (string, time.Time, error)
@ -293,6 +295,7 @@ func (s *apiService) Serve() {
postRestMux.HandleFunc("/rest/db/prio", s.postDBPrio) // folder file [perpage] [page] postRestMux.HandleFunc("/rest/db/prio", s.postDBPrio) // folder file [perpage] [page]
postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores) // folder postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores) // folder
postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder
postRestMux.HandleFunc("/rest/db/revert", s.postDBRevert) // folder
postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay] postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay]
postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore) // folder <body> postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore) // folder <body>
postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // <body> postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // <body>
@ -712,6 +715,17 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter
need := m.NeedSize(folder) need := m.NeedSize(folder)
res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes
if cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly {
// Add statistics for things that have changed locally in a receive
// only folder.
ro := m.ReceiveOnlyChangedSize(folder)
res["receiveOnlyChangedFiles"] = ro.Files
res["receiveOnlyChangedDirectories"] = ro.Directories
res["receiveOnlyChangedSymlinks"] = ro.Symlinks
res["receiveOnlyChangedDeletes"] = ro.Deleted
res["receiveOnlyChangedBytes"] = ro.Bytes
}
res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes
res["state"], res["stateChanged"], err = m.State(folder) res["state"], res["stateChanged"], err = m.State(folder)
@ -748,6 +762,12 @@ func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) {
go s.model.Override(folder) go s.model.Override(folder)
} }
func (s *apiService) postDBRevert(w http.ResponseWriter, r *http.Request) {
var qs = r.URL.Query()
var folder = qs.Get("folder")
go s.model.Revert(folder)
}
func getPagingParams(qs url.Values) (int, int) { func getPagingParams(qs url.Values) (int, int) {
page, err := strconv.Atoi(qs.Get("page")) page, err := strconv.Atoi(qs.Get("page"))
if err != nil || page < 1 { if err != nil || page < 1 {

6
cmd/syncthing/mocked_model_test.go
View File

@ -29,6 +29,8 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model.
func (m *mockedModel) Override(folder string) {} func (m *mockedModel) Override(folder string) {}
func (m *mockedModel) Revert(folder string) {}
func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) { func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
return nil, nil, nil return nil, nil, nil
} }
@ -117,6 +119,10 @@ func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{} return db.Counts{}
} }
func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) { func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {
return 0, false return 0, false
} }

10
gui/default/index.html
View File

@ -301,7 +301,9 @@
<div class="panel-progress" ng-show="folderStatus(folder) == 'scanning' && scanProgress[folder.id] != undefined" ng-attr-style="width: {{scanPercentage(folder.id) | percent}}"></div> <div class="panel-progress" ng-show="folderStatus(folder) == 'scanning' && scanProgress[folder.id] != undefined" ng-attr-style="width: {{scanPercentage(folder.id) | percent}}"></div>
<h4 class="panel-title"> <h4 class="panel-title">
<div class="panel-icon hidden-xs"> <div class="panel-icon hidden-xs">
<span ng-class="[folder.type == 'sendonly' ? 'fas fa-fw fa-lock' : 'fas fa-fw fa-folder']"></span> <span ng-if="folder.type == 'sendreceive'" class="fas fa-fw fa-folder"></span>
<span ng-if="folder.type == 'sendonly'" class="fas fa-fw fa-upload"></span>
<span ng-if="folder.type == 'receiveonly'" class="fas fa-fw fa-download"></span>
</div> </div>
<div class="panel-status pull-right text-{{folderClass(folder)}}" ng-switch="folderStatus(folder)"> <div class="panel-status pull-right text-{{folderClass(folder)}}" ng-switch="folderStatus(folder)">
<span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs">&#9724;</span></span> <span ng-switch-when="paused"><span class="hidden-xs" translate>Paused</span><span class="visible-xs">&#9724;</span></span>
@ -386,9 +388,10 @@
</td> </td>
</tr> </tr>
<tr ng-if="folder.type != 'sendreceive'"> <tr ng-if="folder.type != 'sendreceive'">
<th><span class="fas fa-fw fa-lock"></span>&nbsp;<span translate>Folder Type</span></th> <th><span class="fas fa-fw fa-folder"></span>&nbsp;<span translate>Folder Type</span></th>
<td class="text-right"> <td class="text-right">
<span ng-if="folder.type == 'sendonly'" translate>Send Only</span> <span ng-if="folder.type == 'sendonly'" translate>Send Only</span>
<span ng-if="folder.type == 'receiveonly'" translate>Receive Only</span>
</td> </td>
</tr> </tr>
<tr ng-if="folder.ignorePerms"> <tr ng-if="folder.ignorePerms">
@ -478,6 +481,9 @@
<button type="button" class="btn btn-sm btn-danger pull-left" ng-click="override(folder.id)" ng-if="folderStatus(folder) == 'outofsync' && folder.type == 'sendonly'"> <button type="button" class="btn btn-sm btn-danger pull-left" ng-click="override(folder.id)" ng-if="folderStatus(folder) == 'outofsync' && folder.type == 'sendonly'">
<span class="fas fa-arrow-circle-up"></span>&nbsp;<span translate>Override Changes</span> <span class="fas fa-arrow-circle-up"></span>&nbsp;<span translate>Override Changes</span>
</button> </button>
<button type="button" class="btn btn-sm btn-danger pull-left" ng-click="revert(folder.id)" ng-if="canRevert(folder.id)">
<span class="fa fa-arrow-circle-down"></span>&nbsp;<span translate>Revert Local Changes</span>
</button>
<span class="pull-right"> <span class="pull-right">
<button ng-if="!folder.paused" type="button" class="btn btn-sm btn-default" ng-click="setFolderPause(folder.id, true)"> <button ng-if="!folder.paused" type="button" class="btn btn-sm btn-default" ng-click="setFolderPause(folder.id, true)">
<span class="fas fa-pause"></span>&nbsp;<span translate>Pause</span> <span class="fas fa-pause"></span>&nbsp;<span translate>Pause</span>

16
gui/default/syncthing/core/syncthingController.js
View File

@ -2102,6 +2102,22 @@ angular.module('syncthing.core')
$http.post(urlbase + "/db/override?folder=" + encodeURIComponent(folder)); $http.post(urlbase + "/db/override?folder=" + encodeURIComponent(folder));
}; };
$scope.revert = function (folder) {
$http.post(urlbase + "/db/revert?folder=" + encodeURIComponent(folder));
};
$scope.canRevert = function (folder) {
var f = $scope.model[folder];
if (!f) {
return false;
}
return f.receiveOnlyChangedBytes > 0 ||
f.receiveOnlyChangedDeletes > 0 ||
f.receiveOnlyChangedDirectories > 0 ||
f.receiveOnlyChangedFiles > 0 ||
f.receiveOnlyChangedSymlinks > 0;
};
$scope.advanced = function () { $scope.advanced = function () {
$scope.advancedConfig = angular.copy($scope.config); $scope.advancedConfig = angular.copy($scope.config);
$('#advanced').modal('show'); $('#advanced').modal('show');

2
gui/default/syncthing/folder/editFolderModalView.html
View File

@ -162,8 +162,10 @@
<select class="form-control" ng-model="currentFolder.type"> <select class="form-control" ng-model="currentFolder.type">
<option value="sendreceive" translate>Send &amp; Receive</option> <option value="sendreceive" translate>Send &amp; Receive</option>
<option value="sendonly" translate>Send Only</option> <option value="sendonly" translate>Send Only</option>
<option value="receiveonly" translate>Receive Only</option>
</select> </select>
<p ng-if="currentFolder.type == 'sendonly'" translate class="help-block">Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.</p> <p ng-if="currentFolder.type == 'sendonly'" translate class="help-block">Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.</p>
<p ng-if="currentFolder.type == 'receiveonly'" translate class="help-block">Files are synchronized from the cluster, but any changes made locally will not be sent to other devices.</p>
</div> </div>
<div class="col-md-6 form-group"> <div class="col-md-6 form-group">
<label translate>File Pull Order</label> <label translate>File Pull Order</label>

5
lib/config/foldertype.go
View File

@ -11,6 +11,7 @@ type FolderType int
const ( const (
FolderTypeSendReceive FolderType = iota // default is sendreceive FolderTypeSendReceive FolderType = iota // default is sendreceive
FolderTypeSendOnly FolderTypeSendOnly
FolderTypeReceiveOnly
) )
func (t FolderType) String() string { func (t FolderType) String() string {
@ -19,6 +20,8 @@ func (t FolderType) String() string {
return "sendreceive" return "sendreceive"
case FolderTypeSendOnly: case FolderTypeSendOnly:
return "sendonly" return "sendonly"
case FolderTypeReceiveOnly:
return "receiveonly"
default: default:
return "unknown" return "unknown"
} }
@ -34,6 +37,8 @@ func (t *FolderType) UnmarshalText(bs []byte) error {
*t = FolderTypeSendReceive *t = FolderTypeSendReceive
case "readonly", "sendonly": case "readonly", "sendonly":
*t = FolderTypeSendOnly *t = FolderTypeSendOnly
case "receiveonly":
*t = FolderTypeReceiveOnly
default: default:
*t = FolderTypeSendReceive *t = FolderTypeSendReceive
} }

2
lib/db/leveldb_dbinstance.go
View File

@ -586,7 +586,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) {
if i == 0 { if i == 0 {
if fi, ok := db.getFile(fk); ok { if fi, ok := db.getFile(fk); ok {
meta.addFile(globalDeviceID, fi) meta.addFile(protocol.GlobalDeviceID, fi)
} }
} }
} }

4
lib/db/leveldb_test.go
View File

@ -305,7 +305,7 @@ func TestUpdate0to3(t *testing.T) {
t.Error("Unexpected additional file via sequence", f.FileName()) t.Error("Unexpected additional file via sequence", f.FileName())
return true return true
} }
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) { if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) {
found = true found = true
} else { } else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
@ -330,7 +330,7 @@ func TestUpdate0to3(t *testing.T) {
} }
f := fi.(protocol.FileInfo) f := fi.(protocol.FileInfo)
delete(need, f.Name) delete(need, f.Name)
if !f.IsEquivalent(e, true, true) { if !f.IsEquivalentOptional(e, true, true, 0) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e) t.Errorf("Wrong needed file, got %v, expected %v", f, e)
} }
return true return true

8
lib/db/leveldb_transactions.go
View File

@ -140,11 +140,11 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto
if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok { if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
// A failure to get the file here is surprising and our // A failure to get the file here is surprising and our
// global size data will be incorrect until a restart... // global size data will be incorrect until a restart...
meta.removeFile(globalDeviceID, oldFile) meta.removeFile(protocol.GlobalDeviceID, oldFile)
} }
// Add the new global to the global size counter // Add the new global to the global size counter
meta.addFile(globalDeviceID, newGlobal) meta.addFile(protocol.GlobalDeviceID, newGlobal)
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
t.Put(gk, mustMarshal(&fl)) t.Put(gk, mustMarshal(&fl))
@ -197,7 +197,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
// didn't exist anyway, apparently // didn't exist anyway, apparently
continue continue
} }
meta.removeFile(globalDeviceID, f) meta.removeFile(protocol.GlobalDeviceID, f)
removed = true removed = true
} }
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...) fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
@ -215,7 +215,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok { if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok {
// A failure to get the file here is surprising and our // A failure to get the file here is surprising and our
// global size data will be incorrect until a restart... // global size data will be incorrect until a restart...
meta.addFile(globalDeviceID, f) meta.addFile(protocol.GlobalDeviceID, f)
} }
} }
} }

135
lib/db/meta.go
View File

@ -7,25 +7,30 @@
package db package db
import ( import (
"bytes"
"math/bits"
"time" "time"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/sync"
) )
// like protocol.LocalDeviceID but with 0xf8 in all positions // metadataTracker keeps metadata on a per device, per local flag basis.
var globalDeviceID = protocol.DeviceID{0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8}
type metadataTracker struct { type metadataTracker struct {
mut sync.RWMutex mut sync.RWMutex
counts CountsSet counts CountsSet
indexes map[protocol.DeviceID]int // device ID -> index in counts indexes map[metaKey]int // device ID + local flags -> index in counts
}
type metaKey struct {
dev protocol.DeviceID
flags uint32
} }
func newMetadataTracker() *metadataTracker { func newMetadataTracker() *metadataTracker {
return &metadataTracker{ return &metadataTracker{
mut: sync.NewRWMutex(), mut: sync.NewRWMutex(),
indexes: make(map[protocol.DeviceID]int), indexes: make(map[metaKey]int),
} }
} }
@ -38,7 +43,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error {
// Initialize the index map // Initialize the index map
for i, c := range m.counts.Counts { for i, c := range m.counts.Counts {
m.indexes[protocol.DeviceIDFromBytes(c.DeviceID)] = i m.indexes[metaKey{protocol.DeviceIDFromBytes(c.DeviceID), c.LocalFlags}] = i
} }
return nil return nil
} }
@ -72,14 +77,15 @@ func (m *metadataTracker) fromDB(db *Instance, folder []byte) error {
// countsPtr returns a pointer to the corresponding Counts struct, if // countsPtr returns a pointer to the corresponding Counts struct, if
// necessary allocating one in the process // necessary allocating one in the process
func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts { func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flags uint32) *Counts {
// must be called with the mutex held // must be called with the mutex held
idx, ok := m.indexes[dev] key := metaKey{dev, flags}
idx, ok := m.indexes[key]
if !ok { if !ok {
idx = len(m.counts.Counts) idx = len(m.counts.Counts)
m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:]}) m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flags})
m.indexes[dev] = idx m.indexes[key] = idx
} }
return &m.counts.Counts[idx] return &m.counts.Counts[idx]
} }
@ -87,12 +93,23 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts {
// addFile adds a file to the counts, adjusting the sequence number as // addFile adds a file to the counts, adjusting the sequence number as
// appropriate // appropriate
func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
if f.IsInvalid() { m.mut.Lock()
return
if flags := f.FileLocalFlags(); flags == 0 {
// Account regular files in the zero-flags bucket.
m.addFileLocked(dev, 0, f)
} else {
// Account in flag specific buckets.
eachFlagBit(flags, func(flag uint32) {
m.addFileLocked(dev, flag, f)
})
} }
m.mut.Lock() m.mut.Unlock()
cp := m.countsPtr(dev) }
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, flags)
switch { switch {
case f.IsDeleted(): case f.IsDeleted():
@ -109,18 +126,27 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
if seq := f.SequenceNo(); seq > cp.Sequence { if seq := f.SequenceNo(); seq > cp.Sequence {
cp.Sequence = seq cp.Sequence = seq
} }
m.mut.Unlock()
} }
// removeFile removes a file from the counts // removeFile removes a file from the counts
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
if f.IsInvalid() { m.mut.Lock()
return
if flags := f.FileLocalFlags(); flags == 0 {
// Remove regular files from the zero-flags bucket
m.removeFileLocked(dev, 0, f)
} else {
// Remove from flag specific buckets.
eachFlagBit(flags, func(flag uint32) {
m.removeFileLocked(dev, flag, f)
})
} }
m.mut.Lock() m.mut.Unlock()
cp := m.countsPtr(dev) }
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) {
cp := m.countsPtr(dev, f.FileLocalFlags())
switch { switch {
case f.IsDeleted(): case f.IsDeleted():
@ -153,14 +179,19 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
cp.Symlinks = 0 cp.Symlinks = 0
m.counts.Created = 0 m.counts.Created = 0
} }
m.mut.Unlock()
} }
// resetAll resets all metadata for the given device // resetAll resets all metadata for the given device
func (m *metadataTracker) resetAll(dev protocol.DeviceID) { func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
m.mut.Lock() m.mut.Lock()
*m.countsPtr(dev) = Counts{DeviceID: dev[:]} for i, c := range m.counts.Counts {
if bytes.Equal(c.DeviceID, dev[:]) {
m.counts.Counts[i] = Counts{
DeviceID: c.DeviceID,
LocalFlags: c.LocalFlags,
}
}
}
m.mut.Unlock() m.mut.Unlock()
} }
@ -169,23 +200,30 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) {
func (m *metadataTracker) resetCounts(dev protocol.DeviceID) { func (m *metadataTracker) resetCounts(dev protocol.DeviceID) {
m.mut.Lock() m.mut.Lock()
c := m.countsPtr(dev) for i, c := range m.counts.Counts {
c.Bytes = 0 if bytes.Equal(c.DeviceID, dev[:]) {
c.Deleted = 0 m.counts.Counts[i] = Counts{
c.Directories = 0 DeviceID: c.DeviceID,
c.Files = 0 Sequence: c.Sequence,
c.Symlinks = 0 LocalFlags: c.LocalFlags,
// c.Sequence deliberately untouched }
}
}
m.mut.Unlock() m.mut.Unlock()
} }
// Counts returns the counts for the given device ID // Counts returns the counts for the given device ID and flag. `flag` should
func (m *metadataTracker) Counts(dev protocol.DeviceID) Counts { // be zero or have exactly one bit set.
func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts {
if bits.OnesCount32(flag) > 1 {
panic("incorrect usage: set at most one bit in flag")
}
m.mut.RLock() m.mut.RLock()
defer m.mut.RUnlock() defer m.mut.RUnlock()
idx, ok := m.indexes[dev] idx, ok := m.indexes[metaKey{dev, flag}]
if !ok { if !ok {
return Counts{} return Counts{}
} }
@ -198,7 +236,7 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 {
m.mut.Lock() m.mut.Lock()
defer m.mut.Unlock() defer m.mut.Unlock()
c := m.countsPtr(dev) c := m.countsPtr(dev, 0)
c.Sequence++ c.Sequence++
return c.Sequence return c.Sequence
} }
@ -206,21 +244,26 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 {
// devices returns the list of devices tracked, excluding the local device // devices returns the list of devices tracked, excluding the local device
// (which we don't know the ID of) // (which we don't know the ID of)
func (m *metadataTracker) devices() []protocol.DeviceID { func (m *metadataTracker) devices() []protocol.DeviceID {
devs := make([]protocol.DeviceID, 0, len(m.counts.Counts)) devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts))
m.mut.RLock() m.mut.RLock()
for _, dev := range m.counts.Counts { for _, dev := range m.counts.Counts {
if dev.Sequence > 0 { if dev.Sequence > 0 {
id := protocol.DeviceIDFromBytes(dev.DeviceID) id := protocol.DeviceIDFromBytes(dev.DeviceID)
if id == globalDeviceID || id == protocol.LocalDeviceID { if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID {
continue continue
} }
devs = append(devs, id) devs[id] = struct{}{}
} }
} }
m.mut.RUnlock() m.mut.RUnlock()
return devs devList := make([]protocol.DeviceID, 0, len(devs))
for dev := range devs {
devList = append(devList, dev)
}
return devList
} }
func (m *metadataTracker) Created() time.Time { func (m *metadataTracker) Created() time.Time {
@ -234,3 +277,19 @@ func (m *metadataTracker) SetCreated() {
m.counts.Created = time.Now().UnixNano() m.counts.Created = time.Now().UnixNano()
m.mut.Unlock() m.mut.Unlock()
} }
// eachFlagBit calls the function once for every bit that is set in flags
func eachFlagBit(flags uint32, fn func(flag uint32)) {
// Test each bit from the right, as long as there are bits left in the
// flag set. Clear any bits found and stop testing as soon as there are
// no more bits set.
currentBit := uint32(1 << 0)
for flags != 0 {
if flags&currentBit != 0 {
fn(currentBit)
flags &^= currentBit
}
currentBit <<= 1
}
}

82
lib/db/meta_test.go Normal file
View File

@ -0,0 +1,82 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package db
import (
"math/bits"
"sort"
"testing"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestEachFlagBit(t *testing.T) {
cases := []struct {
flags uint32
iterations int
}{
{0, 0},
{1<<0 | 1<<3, 2},
{1 << 0, 1},
{1 << 31, 1},
{1<<10 | 1<<20 | 1<<30, 3},
}
for _, tc := range cases {
var flags uint32
iterations := 0
eachFlagBit(tc.flags, func(f uint32) {
iterations++
flags |= f
if bits.OnesCount32(f) != 1 {
t.Error("expected exactly one bit to be set in every call")
}
})
if flags != tc.flags {
t.Errorf("expected 0x%x flags, got 0x%x", tc.flags, flags)
}
if iterations != tc.iterations {
t.Errorf("expected %d iterations, got %d", tc.iterations, iterations)
}
}
}
func TestMetaDevices(t *testing.T) {
d1 := protocol.DeviceID{1}
d2 := protocol.DeviceID{2}
meta := newMetadataTracker()
meta.addFile(d1, protocol.FileInfo{Sequence: 1})
meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 1})
meta.addFile(d2, protocol.FileInfo{Sequence: 2, LocalFlags: 2})
meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1})
// There are five device/flags combos
if l := len(meta.counts.Counts); l < 5 {
t.Error("expected at least five buckets, not", l)
}
// There are only two non-local devices
devs := meta.devices()
if l := len(devs); l != 2 {
t.Fatal("expected two devices, not", l)
}
// Check that we got the two devices we expect
sort.Slice(devs, func(a, b int) bool {
return devs[a].Compare(devs[b]) == -1
})
if devs[0] != d1 {
t.Error("first device should be d1")
}
if devs[1] != d2 {
t.Error("second device should be d2")
}
}

18
lib/db/set.go
View File

@ -37,8 +37,12 @@ type FileSet struct {
type FileIntf interface { type FileIntf interface {
FileSize() int64 FileSize() int64
FileName() string FileName() string
FileLocalFlags() uint32
IsDeleted() bool IsDeleted() bool
IsInvalid() bool IsInvalid() bool
IsIgnored() bool
IsUnsupported() bool
MustRescan() bool
IsDirectory() bool IsDirectory() bool
IsSymlink() bool IsSymlink() bool
HasPermissionBits() bool HasPermissionBits() bool
@ -248,15 +252,23 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
} }
func (s *FileSet) Sequence(device protocol.DeviceID) int64 { func (s *FileSet) Sequence(device protocol.DeviceID) int64 {
return s.meta.Counts(device).Sequence return s.meta.Counts(device, 0).Sequence
} }
func (s *FileSet) LocalSize() Counts { func (s *FileSet) LocalSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID) local := s.meta.Counts(protocol.LocalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
return local.Add(recvOnlyChanged)
}
func (s *FileSet) ReceiveOnlyChangedSize() Counts {
return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly)
} }
func (s *FileSet) GlobalSize() Counts { func (s *FileSet) GlobalSize() Counts {
return s.meta.Counts(globalDeviceID) global := s.meta.Counts(protocol.GlobalDeviceID, 0)
recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly)
return global.Add(recvOnlyChanged)
} }
func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID { func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {

118
lib/db/set_test.go
View File

@ -906,7 +906,7 @@ func TestWithHaveSequence(t *testing.T) {
i := 2 i := 2
s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], false, false) { if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
} }
i++ i++
@ -917,7 +917,7 @@ func TestWithHaveSequence(t *testing.T) {
func TestIssue4925(t *testing.T) { func TestIssue4925(t *testing.T) {
ldb := db.OpenMemory() ldb := db.OpenMemory()
folder := "test)" folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
localHave := fileList{ localHave := fileList{
@ -955,7 +955,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Error("Expected 1 local need, got", need) t.Error("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(remote0Have[0], false, false) { } else if !need[0].IsEquivalent(remote0Have[0]) {
t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0]) t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
} }
@ -981,7 +981,7 @@ func TestMoveGlobalBack(t *testing.T) {
if need := needList(s, remoteDevice0); len(need) != 1 { if need := needList(s, remoteDevice0); len(need) != 1 {
t.Error("Expected 1 need for remote 0, got", need) t.Error("Expected 1 need for remote 0, got", need)
} else if !need[0].IsEquivalent(localHave[0], false, false) { } else if !need[0].IsEquivalent(localHave[0]) {
t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0]) t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
} }
@ -1017,7 +1017,7 @@ func TestIssue5007(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need) t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], false, false) { } else if !need[0].IsEquivalent(fs[0]) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
} }
@ -1052,7 +1052,7 @@ func TestNeedDeleted(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need) t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], false, false) { } else if !need[0].IsEquivalent(fs[0]) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
} }
@ -1065,6 +1065,110 @@ func TestNeedDeleted(t *testing.T) {
} }
} }
func TestReceiveOnlyAccounting(t *testing.T) {
ldb := db.OpenMemory()
folder := "test"
s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
local := protocol.DeviceID{1}
remote := protocol.DeviceID{2}
// Three files that have been created by the remote device
version := protocol.Vector{Counters: []protocol.Counter{{ID: remote.Short(), Value: 1}}}
files := fileList{
protocol.FileInfo{Name: "f1", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f2", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f3", Size: 10, Sequence: 1, Version: version},
}
// We have synced them locally
replace(s, protocol.LocalDeviceID, files)
replace(s, remote, files)
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files initially, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
t.Fatal("expected 30 local bytes initially, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files initially, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
t.Fatal("expected 30 global bytes initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
t.Fatal("expected 0 receive only changed files initially, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes initially, not", n)
}
// Detected a local change in a receive only folder
changed := files[0]
changed.Version = changed.Version.Update(local.Short())
changed.Size = 100
changed.ModifiedBy = local.Short()
changed.LocalFlags = protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
// Check that we see the files
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files after local change, not", n)
}
if n := s.LocalSize().Bytes; n != 120 {
t.Fatal("expected 120 local bytes after local change, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files after local change, not", n)
}
if n := s.GlobalSize().Bytes; n != 120 {
t.Fatal("expected 120 global bytes after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 1 {
t.Fatal("expected 1 receive only changed file after local change, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 {
t.Fatal("expected 100 receive only changed btyes after local change, not", n)
}
// Fake a revert. That's a two step process, first converting our
// changed file into a less preferred variant, then pulling down the old
// version.
changed.Version = protocol.Vector{}
changed.LocalFlags &^= protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{files[0]})
// Check that we see the files, same data as initially
if n := s.LocalSize().Files; n != 3 {
t.Fatal("expected 3 local files after revert, not", n)
}
if n := s.LocalSize().Bytes; n != 30 {
t.Fatal("expected 30 local bytes after revert, not", n)
}
if n := s.GlobalSize().Files; n != 3 {
t.Fatal("expected 3 global files after revert, not", n)
}
if n := s.GlobalSize().Bytes; n != 30 {
t.Fatal("expected 30 global bytes after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Files; n != 0 {
t.Fatal("expected 0 receive only changed files after revert, not", n)
}
if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes after revert, not", n)
}
}
func TestNeedAfterUnignore(t *testing.T) { func TestNeedAfterUnignore(t *testing.T) {
ldb := db.OpenMemory() ldb := db.OpenMemory()
@ -1090,7 +1194,7 @@ func TestNeedAfterUnignore(t *testing.T) {
if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need) t.Fatal("Expected one local need, got", need)
} else if !need[0].IsEquivalent(remote, false, false) { } else if !need[0].IsEquivalent(remote) {
t.Fatalf("Got %v, expected %v", need[0], remote) t.Fatalf("Got %v, expected %v", need[0], remote)
} }
} }

25
lib/db/structs.go
View File

@ -40,6 +40,10 @@ func (f FileInfoTruncated) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0 return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0
} }
func (f FileInfoTruncated) IsUnsupported() bool {
return f.LocalFlags&protocol.FlagLocalUnsupported != 0
}
func (f FileInfoTruncated) IsIgnored() bool { func (f FileInfoTruncated) IsIgnored() bool {
return f.LocalFlags&protocol.FlagLocalIgnored != 0 return f.LocalFlags&protocol.FlagLocalIgnored != 0
} }
@ -48,6 +52,10 @@ func (f FileInfoTruncated) MustRescan() bool {
return f.LocalFlags&protocol.FlagLocalMustRescan != 0 return f.LocalFlags&protocol.FlagLocalMustRescan != 0
} }
func (f FileInfoTruncated) IsReceiveOnlyChanged() bool {
return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0
}
func (f FileInfoTruncated) IsDirectory() bool { func (f FileInfoTruncated) IsDirectory() bool {
return f.Type == protocol.FileInfoTypeDirectory return f.Type == protocol.FileInfoTypeDirectory
} }
@ -86,6 +94,10 @@ func (f FileInfoTruncated) FileName() string {
return f.Name return f.Name
} }
func (f FileInfoTruncated) FileLocalFlags() uint32 {
return f.LocalFlags
}
func (f FileInfoTruncated) ModTime() time.Time { func (f FileInfoTruncated) ModTime() time.Time {
return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
} }
@ -110,3 +122,16 @@ func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protoco
LocalFlags: protocol.FlagLocalIgnored, LocalFlags: protocol.FlagLocalIgnored,
} }
} }
func (c Counts) Add(other Counts) Counts {
return Counts{
Files: c.Files + other.Files,
Directories: c.Directories + other.Directories,
Symlinks: c.Symlinks + other.Symlinks,
Deleted: c.Deleted + other.Deleted,
Bytes: c.Bytes + other.Bytes,
Sequence: c.Sequence + other.Sequence,
DeviceID: protocol.EmptyDeviceID[:],
LocalFlags: c.LocalFlags | other.LocalFlags,
}
}

116
lib/db/structs.pb.go
View File

@ -91,6 +91,7 @@ type Counts struct {
Bytes int64 `protobuf:"varint,5,opt,name=bytes,proto3" json:"bytes,omitempty"` Bytes int64 `protobuf:"varint,5,opt,name=bytes,proto3" json:"bytes,omitempty"`
Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"` Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"`
DeviceID []byte `protobuf:"bytes,17,opt,name=deviceID,proto3" json:"deviceID,omitempty"` DeviceID []byte `protobuf:"bytes,17,opt,name=deviceID,proto3" json:"deviceID,omitempty"`
LocalFlags uint32 `protobuf:"varint,18,opt,name=localFlags,proto3" json:"localFlags,omitempty"`
} }
func (m *Counts) Reset() { *m = Counts{} } func (m *Counts) Reset() { *m = Counts{} }
@ -357,6 +358,13 @@ func (m *Counts) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintStructs(dAtA, i, uint64(len(m.DeviceID))) i = encodeVarintStructs(dAtA, i, uint64(len(m.DeviceID)))
i += copy(dAtA[i:], m.DeviceID) i += copy(dAtA[i:], m.DeviceID)
} }
if m.LocalFlags != 0 {
dAtA[i] = 0x90
i++
dAtA[i] = 0x1
i++
i = encodeVarintStructs(dAtA, i, uint64(m.LocalFlags))
}
return i, nil return i, nil
} }
@ -526,6 +534,9 @@ func (m *Counts) ProtoSize() (n int) {
if l > 0 { if l > 0 {
n += 2 + l + sovStructs(uint64(l)) n += 2 + l + sovStructs(uint64(l))
} }
if m.LocalFlags != 0 {
n += 2 + sovStructs(uint64(m.LocalFlags))
}
return n return n
} }
@ -1312,6 +1323,25 @@ func (m *Counts) Unmarshal(dAtA []byte) error {
m.DeviceID = []byte{} m.DeviceID = []byte{}
} }
iNdEx = postIndex iNdEx = postIndex
case 18:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType)
}
m.LocalFlags = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.LocalFlags |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:]) skippy, err := skipStructs(dAtA[iNdEx:])
@ -1541,47 +1571,47 @@ var (
func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) } func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) }
var fileDescriptorStructs = []byte{ var fileDescriptorStructs = []byte{
// 663 bytes of a gzipped FileDescriptorProto // 671 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x40, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6b, 0xdb, 0x4c,
0x10, 0xb6, 0x62, 0xf9, 0x6f, 0x6c, 0xa7, 0xc9, 0x12, 0x82, 0x30, 0xd4, 0x16, 0x86, 0x82, 0x28, 0x10, 0xb6, 0x62, 0xf9, 0x6b, 0x6c, 0xe7, 0x4d, 0x96, 0x10, 0x84, 0xe1, 0xb5, 0x85, 0xa1, 0x20,
0xd4, 0x6e, 0x13, 0x7a, 0x69, 0x6f, 0x6a, 0x08, 0x18, 0x4a, 0x5b, 0xd6, 0x21, 0xa7, 0x82, 0xd1, 0x0a, 0xb5, 0xdb, 0x84, 0x5e, 0xda, 0x9b, 0x1a, 0x02, 0x86, 0xd2, 0x96, 0x75, 0xc8, 0xa9, 0x60,
0xcf, 0xda, 0x59, 0x22, 0x6b, 0x1d, 0xed, 0x3a, 0x41, 0x79, 0x92, 0x1e, 0xf3, 0x30, 0x3d, 0xe4, 0xf4, 0xb1, 0x76, 0x96, 0xc8, 0x5a, 0x47, 0xbb, 0x4e, 0x50, 0x7e, 0x49, 0x8f, 0xf9, 0x39, 0x39,
0xd8, 0x73, 0x0f, 0x26, 0x75, 0x2f, 0x7d, 0x8c, 0xb2, 0xbb, 0x92, 0xa2, 0xf6, 0xd4, 0xde, 0xe6, 0xf6, 0xdc, 0x83, 0x49, 0xdd, 0x1e, 0xfa, 0x33, 0xca, 0xee, 0x4a, 0x8a, 0x9a, 0x53, 0x7b, 0x9b,
0x9b, 0x9f, 0x9d, 0x6f, 0x66, 0xbe, 0x85, 0x2e, 0x17, 0xc9, 0x3a, 0x10, 0x7c, 0xb4, 0x4a, 0x98, 0x67, 0x3e, 0x76, 0x9e, 0x99, 0x79, 0x16, 0xba, 0x5c, 0x24, 0xeb, 0x40, 0xf0, 0xd1, 0x2a, 0x61,
0x60, 0x68, 0x27, 0xf4, 0x7b, 0x2f, 0x16, 0x54, 0x5c, 0xac, 0xfd, 0x51, 0xc0, 0x96, 0xe3, 0x05, 0x82, 0xa1, 0x9d, 0xd0, 0xef, 0xbd, 0x58, 0x50, 0x71, 0xb1, 0xf6, 0x47, 0x01, 0x5b, 0x8e, 0x17,
0x5b, 0xb0, 0xb1, 0x0a, 0xf9, 0xeb, 0xb9, 0x42, 0x0a, 0x28, 0x4b, 0x97, 0xf4, 0x5e, 0x97, 0xd2, 0x6c, 0xc1, 0xc6, 0x2a, 0xe4, 0xaf, 0xe7, 0x0a, 0x29, 0xa0, 0x2c, 0x5d, 0xd2, 0x7b, 0x5d, 0x4a,
0x79, 0x1a, 0x07, 0xe2, 0x82, 0xc6, 0x8b, 0x92, 0x15, 0x51, 0x5f, 0xbf, 0x10, 0xb0, 0x68, 0xec, 0xe7, 0x69, 0x1c, 0x88, 0x0b, 0x1a, 0x2f, 0x4a, 0x56, 0x44, 0x7d, 0xfd, 0x42, 0xc0, 0xa2, 0xb1,
0x93, 0x95, 0x2e, 0x1b, 0x5e, 0x41, 0xfb, 0x94, 0x46, 0xe4, 0x9c, 0x24, 0x9c, 0xb2, 0x18, 0xbd, 0x4f, 0x56, 0xba, 0x6c, 0x78, 0x05, 0xed, 0x53, 0x1a, 0x91, 0x73, 0x92, 0x70, 0xca, 0x62, 0xf4,
0x84, 0xc6, 0xb5, 0x36, 0x2d, 0xc3, 0x36, 0x9c, 0xf6, 0xd1, 0xde, 0x28, 0x2f, 0x1a, 0x9d, 0x93, 0x12, 0x1a, 0xd7, 0xda, 0xb4, 0x0c, 0xdb, 0x70, 0xda, 0x47, 0x7b, 0xa3, 0xbc, 0x68, 0x74, 0x4e,
0x40, 0xb0, 0xc4, 0x35, 0xef, 0x37, 0x83, 0x0a, 0xce, 0xd3, 0xd0, 0x21, 0xd4, 0x43, 0x72, 0x4d, 0x02, 0xc1, 0x12, 0xd7, 0xbc, 0xdf, 0x0c, 0x2a, 0x38, 0x4f, 0x43, 0x87, 0x50, 0x0f, 0xc9, 0x35,
0x03, 0x62, 0xed, 0xd8, 0x86, 0xd3, 0xc1, 0x19, 0x42, 0x16, 0x34, 0x68, 0x7c, 0xed, 0x45, 0x34, 0x0d, 0x88, 0xb5, 0x63, 0x1b, 0x4e, 0x07, 0x67, 0x08, 0x59, 0xd0, 0xa0, 0xf1, 0xb5, 0x17, 0xd1,
0xb4, 0xaa, 0xb6, 0xe1, 0x34, 0x71, 0x0e, 0x87, 0xa7, 0xd0, 0xce, 0xda, 0xbd, 0xa7, 0x5c, 0xa0, 0xd0, 0xaa, 0xda, 0x86, 0xd3, 0xc4, 0x39, 0x1c, 0x9e, 0x42, 0x3b, 0x6b, 0xf7, 0x9e, 0x72, 0x81,
0x57, 0xd0, 0xcc, 0xde, 0xe2, 0x96, 0x61, 0x57, 0x9d, 0xf6, 0xd1, 0x93, 0x51, 0xe8, 0x8f, 0x4a, 0x5e, 0x41, 0x33, 0x7b, 0x8b, 0x5b, 0x86, 0x5d, 0x75, 0xda, 0x47, 0xff, 0x8d, 0x42, 0x7f, 0x54,
0xac, 0xb2, 0x96, 0x45, 0xda, 0x1b, 0xf3, 0xcb, 0xdd, 0xa0, 0x32, 0x7c, 0x30, 0x61, 0x5f, 0x66, 0x62, 0x95, 0xb5, 0x2c, 0xd2, 0xde, 0x98, 0x5f, 0xee, 0x06, 0x95, 0xe1, 0x83, 0x09, 0xfb, 0x32,
0x4d, 0xe2, 0x39, 0x3b, 0x4b, 0xd6, 0x71, 0xe0, 0x09, 0x12, 0x22, 0x04, 0x66, 0xec, 0x2d, 0x89, 0x6b, 0x12, 0xcf, 0xd9, 0x59, 0xb2, 0x8e, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, 0x6f, 0x49,
0xa2, 0xdf, 0xc2, 0xca, 0x46, 0xcf, 0xc1, 0x14, 0xe9, 0x4a, 0x33, 0xdc, 0x3d, 0x3a, 0x7c, 0x1c, 0x14, 0xfd, 0x16, 0x56, 0x36, 0x7a, 0x0e, 0xa6, 0x48, 0x57, 0x9a, 0xe1, 0xee, 0xd1, 0xe1, 0xe3,
0xa9, 0x28, 0x4f, 0x57, 0x04, 0xab, 0x1c, 0x59, 0xcf, 0xe9, 0x2d, 0x51, 0xa4, 0xab, 0x58, 0xd9, 0x48, 0x45, 0x79, 0xba, 0x22, 0x58, 0xe5, 0xc8, 0x7a, 0x4e, 0x6f, 0x89, 0x22, 0x5d, 0xc5, 0xca,
0xc8, 0x86, 0xf6, 0x8a, 0x24, 0x4b, 0xca, 0x35, 0x4b, 0xd3, 0x36, 0x9c, 0x2e, 0x2e, 0xbb, 0xd0, 0x46, 0x36, 0xb4, 0x57, 0x24, 0x59, 0x52, 0xae, 0x59, 0x9a, 0xb6, 0xe1, 0x74, 0x71, 0xd9, 0x85,
0x53, 0x80, 0x25, 0x0b, 0xe9, 0x9c, 0x92, 0x70, 0xc6, 0xad, 0x9a, 0xaa, 0x6d, 0xe5, 0x9e, 0xa9, 0xfe, 0x07, 0x58, 0xb2, 0x90, 0xce, 0x29, 0x09, 0x67, 0xdc, 0xaa, 0xa9, 0xda, 0x56, 0xee, 0x99,
0x5c, 0x46, 0x48, 0x22, 0x22, 0x48, 0x68, 0xd5, 0xf5, 0x32, 0x32, 0x88, 0x9c, 0xc7, 0x35, 0x35, 0xca, 0x65, 0x84, 0x24, 0x22, 0x82, 0x84, 0x56, 0x5d, 0x2f, 0x23, 0x83, 0xc8, 0x79, 0x5c, 0x53,
0x64, 0xc4, 0xdd, 0xdd, 0x6e, 0x06, 0x80, 0xbd, 0x9b, 0x89, 0xf6, 0x16, 0x6b, 0x43, 0xcf, 0x60, 0x43, 0x46, 0xdc, 0xdd, 0xed, 0x66, 0x00, 0xd8, 0xbb, 0x99, 0x68, 0x6f, 0xb1, 0x36, 0xf4, 0x0c,
0x37, 0x66, 0xb3, 0x32, 0x8f, 0xa6, 0x7a, 0xaa, 0x1b, 0xb3, 0x4f, 0x25, 0x26, 0xa5, 0x0b, 0xb6, 0x76, 0x63, 0x36, 0x2b, 0xf3, 0x68, 0xaa, 0xa7, 0xba, 0x31, 0xfb, 0x54, 0x62, 0x52, 0xba, 0x60,
0xfe, 0xed, 0x82, 0x3d, 0x68, 0x72, 0x72, 0xb5, 0x26, 0x71, 0x40, 0x2c, 0x50, 0xcc, 0x0b, 0x8c, 0xeb, 0xef, 0x2e, 0xd8, 0x83, 0x26, 0x27, 0x57, 0x6b, 0x12, 0x07, 0xc4, 0x02, 0xc5, 0xbc, 0xc0,
0x06, 0xd0, 0x2e, 0xe6, 0x8a, 0xb9, 0xd5, 0xb6, 0x0d, 0xa7, 0x86, 0x8b, 0x51, 0x3f, 0x70, 0xf4, 0x68, 0x00, 0xed, 0x62, 0xae, 0x98, 0x5b, 0x6d, 0xdb, 0x70, 0x6a, 0xb8, 0x18, 0xf5, 0x03, 0x47,
0xb9, 0x94, 0xe0, 0xa7, 0x56, 0xc7, 0x36, 0x1c, 0xd3, 0x7d, 0x2b, 0x1b, 0x7c, 0xdf, 0x0c, 0x8e, 0x9f, 0x4b, 0x09, 0x7e, 0x6a, 0x75, 0x6c, 0xc3, 0x31, 0xdd, 0xb7, 0xb2, 0xc1, 0xb7, 0xcd, 0xe0,
0xff, 0x43, 0x93, 0xa3, 0xe9, 0x05, 0x4b, 0xc4, 0xe4, 0xe4, 0xf1, 0x75, 0x37, 0x45, 0x63, 0x00, 0xf8, 0x1f, 0x34, 0x39, 0x9a, 0x5e, 0xb0, 0x44, 0x4c, 0x4e, 0x1e, 0x5f, 0x77, 0x53, 0x34, 0x06,
0x3f, 0x62, 0xc1, 0xe5, 0x4c, 0x9d, 0xa4, 0x2b, 0xbb, 0xbb, 0x7b, 0xdb, 0xcd, 0xa0, 0x83, 0xbd, 0xf0, 0x23, 0x16, 0x5c, 0xce, 0xd4, 0x49, 0xba, 0xb2, 0xbb, 0xbb, 0xb7, 0xdd, 0x0c, 0x3a, 0xd8,
0x1b, 0x57, 0x06, 0xa6, 0xf4, 0x96, 0xe0, 0x96, 0x9f, 0x9b, 0x72, 0x49, 0x3c, 0x5d, 0x46, 0x34, 0xbb, 0x71, 0x65, 0x60, 0x4a, 0x6f, 0x09, 0x6e, 0xf9, 0xb9, 0x29, 0x97, 0xc4, 0xd3, 0x65, 0x44,
0xbe, 0x9c, 0x09, 0x2f, 0x59, 0x10, 0x61, 0xed, 0x2b, 0x1d, 0x74, 0x33, 0xef, 0x99, 0x72, 0xca, 0xe3, 0xcb, 0x99, 0xf0, 0x92, 0x05, 0x11, 0xd6, 0xbe, 0xd2, 0x41, 0x37, 0xf3, 0x9e, 0x29, 0xa7,
0x83, 0x46, 0x2c, 0xf0, 0xa2, 0xd9, 0x3c, 0xf2, 0x16, 0xdc, 0xfa, 0xd5, 0x50, 0x17, 0x05, 0xe5, 0x3c, 0x68, 0xc4, 0x02, 0x2f, 0x9a, 0xcd, 0x23, 0x6f, 0xc1, 0xad, 0x5f, 0x0d, 0x75, 0x51, 0x50,
0x3b, 0x95, 0xae, 0x4c, 0x62, 0x5f, 0x0d, 0xa8, 0xbf, 0x63, 0xeb, 0x58, 0x70, 0x74, 0x00, 0xb5, 0xbe, 0x53, 0xe9, 0xca, 0x24, 0xf6, 0xd3, 0x80, 0xfa, 0x3b, 0xb6, 0x8e, 0x05, 0x47, 0x07, 0x50,
0x39, 0x8d, 0x08, 0x57, 0xc2, 0xaa, 0x61, 0x0d, 0xe4, 0x43, 0x21, 0x4d, 0xd4, 0x5a, 0x29, 0xe1, 0x9b, 0xd3, 0x88, 0x70, 0x25, 0xac, 0x1a, 0xd6, 0x40, 0x3e, 0x14, 0xd2, 0x44, 0xad, 0x95, 0x12,
0x4a, 0x60, 0x35, 0x5c, 0x76, 0xa9, 0xed, 0xea, 0xde, 0x5c, 0x69, 0xaa, 0x86, 0x0b, 0x5c, 0x96, 0xae, 0x04, 0x56, 0xc3, 0x65, 0x97, 0xda, 0xae, 0xee, 0xcd, 0x95, 0xa6, 0x6a, 0xb8, 0xc0, 0x65,
0x85, 0xa9, 0x42, 0x85, 0x2c, 0x0e, 0xa0, 0xe6, 0xa7, 0x82, 0xe4, 0x52, 0xd2, 0xe0, 0x8f, 0x4b, 0x59, 0x98, 0x2a, 0x54, 0xc8, 0xe2, 0x00, 0x6a, 0x7e, 0x2a, 0x48, 0x2e, 0x25, 0x0d, 0xfe, 0xb8,
0xd5, 0xff, 0xba, 0x54, 0x0f, 0x9a, 0xfa, 0xe7, 0x4d, 0x4e, 0xd4, 0xcc, 0x1d, 0x5c, 0xe0, 0xe1, 0x54, 0xfd, 0xc9, 0xa5, 0x7a, 0xd0, 0xd4, 0x3f, 0x6f, 0x72, 0xa2, 0x66, 0xee, 0xe0, 0x02, 0xa3,
0x47, 0x68, 0xe9, 0x29, 0xa6, 0x44, 0x20, 0x07, 0xea, 0x81, 0x02, 0xd9, 0x6f, 0x03, 0xf9, 0xdb, 0x3e, 0x94, 0x46, 0xb3, 0xd0, 0xd3, 0x61, 0x87, 0x1f, 0xa1, 0xa5, 0xa7, 0x9c, 0x12, 0x81, 0x1c,
0x74, 0x38, 0x53, 0x46, 0x16, 0x97, 0xf4, 0x82, 0x84, 0xc8, 0x5f, 0xa5, 0x06, 0xab, 0xe2, 0x1c, 0xa8, 0x07, 0x0a, 0x64, 0xbf, 0x11, 0xe4, 0x6f, 0xd4, 0xe1, 0x4c, 0x39, 0x59, 0x5c, 0xd2, 0x0f,
0xba, 0x07, 0xf7, 0x3f, 0xfa, 0x95, 0xfb, 0x6d, 0xdf, 0xf8, 0xb6, 0xed, 0x1b, 0x0f, 0xdb, 0x7e, 0x12, 0x22, 0x7f, 0x9d, 0x1a, 0xbc, 0x8a, 0x73, 0xe8, 0x1e, 0xdc, 0x7f, 0xef, 0x57, 0xee, 0xb7,
0xe5, 0xee, 0x67, 0xdf, 0xf0, 0xeb, 0xea, 0x96, 0xc7, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x7d, 0xe3, 0xeb, 0xb6, 0x6f, 0x3c, 0x6c, 0xfb, 0x95, 0xbb, 0x1f, 0x7d, 0xc3, 0xaf, 0xab, 0x5b,
0x4b, 0x16, 0x44, 0xcd, 0x04, 0x00, 0x00, 0x1f, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x4d, 0xd7, 0x14, 0xed, 0x04, 0x00, 0x00,
} }

15
lib/db/structs.proto
View File

@ -46,13 +46,14 @@ message FileInfoTruncated {
// For each folder and device we keep one of these to track the current // For each folder and device we keep one of these to track the current
// counts and sequence. We also keep one for the global state of the folder. // counts and sequence. We also keep one for the global state of the folder.
message Counts { message Counts {
int32 files = 1; int32 files = 1;
int32 directories = 2; int32 directories = 2;
int32 symlinks = 3; int32 symlinks = 3;
int32 deleted = 4; int32 deleted = 4;
int64 bytes = 5; int64 bytes = 5;
int64 sequence = 6; // zero for the global state int64 sequence = 6; // zero for the global state
bytes deviceID = 17; // device ID for remote devices, or special values for local/global bytes deviceID = 17; // device ID for remote devices, or special values for local/global
uint32 localFlags = 18; // the local flag for this count bucket
} }
message CountsSet { message CountsSet {

5
lib/model/folder.go
View File

@ -27,6 +27,7 @@ var errWatchNotStarted = errors.New("not started")
type folder struct { type folder struct {
stateTracker stateTracker
config.FolderConfiguration config.FolderConfiguration
localFlags uint32
model *Model model *Model
shortID protocol.ShortID shortID protocol.ShortID
@ -175,6 +176,8 @@ func (f *folder) BringToFront(string) {}
func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}
func (f *folder) DelayScan(next time.Duration) { func (f *folder) DelayScan(next time.Duration) {
f.Delay(next) f.Delay(next)
} }
@ -263,7 +266,7 @@ func (f *folder) getHealthError() error {
} }
func (f *folder) scanSubdirs(subDirs []string) error { func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil { if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs, f.localFlags); err != nil {
// Potentially sets the error twice, once in the scanner just // Potentially sets the error twice, once in the scanner just
// by doing a check, and once here, if the error returned is // by doing a check, and once here, if the error returned is
// the same one as returned by CheckHealth, though // the same one as returned by CheckHealth, though

210
lib/model/folder_recvonly.go Normal file
View File

@ -0,0 +1,210 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"sort"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/versioner"
)
func init() {
folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder
}
/*
receiveOnlyFolder is a folder that does not propagate local changes outward.
It does this by the following general mechanism (not all of which is
implemted in this file):
- Local changes are scanned and versioned as usual, but get the
FlagLocalReceiveOnly bit set.
- When changes are sent to the cluster this bit gets converted to the
Invalid bit (like all other local flags, currently) and also the Version
gets set to the empty version. The reason for clearing the Version is to
ensure that other devices will not consider themselves out of date due to
our change.
- The database layer accounts sizes per flag bit, so we can know how many
files have been changed locally. We use this to trigger a "Revert" option
on the folder when the amount of locally changed data is nonzero.
- To revert we take the files which have changed and reset their version
counter down to zero. The next pull will replace our changed version with
the globally latest. As this is a user-initiated operation we do not cause
conflict copies when reverting.
- When pulling normally (i.e., not in the revert case) with local changes,
normal conflict resolution will apply. Conflict copies will be created,
but not propagated outwards (because receive only, right).
Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that
sets an extra bit on local changes and has a Revert method.
*/
type receiveOnlyFolder struct {
*sendReceiveFolder
}
func newReceiveOnlyFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
sr := newSendReceiveFolder(model, cfg, ver, fs).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr}
}
func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {
f.setState(FolderScanning)
defer f.setState(FolderIdle)
// XXX: This *really* should be given to us in the constructor...
f.model.fmut.RLock()
ignores := f.model.folderIgnores[f.folderID]
f.model.fmut.RUnlock()
delQueue := &deleteQueue{
handler: f, // for the deleteFile and deleteDir methods
ignores: ignores,
}
batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles)
batchSizeBytes := 0
fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in
// receive only mode.
return true
}
if len(fi.Version.Counters) == 1 && fi.Version.Counters[0].ID == f.shortID {
// We are the only device mentioned in the version vector so the
// file must originate here. A revert then means to delete it.
// We'll delete files directly, directories get queued and
// handled below.
handled, err := delQueue.handle(fi)
if err != nil {
l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
return true // continue
}
if !handled {
return true // continue
}
fi = protocol.FileInfo{
Name: fi.Name,
Type: fi.Type,
ModifiedS: fi.ModifiedS,
ModifiedNs: fi.ModifiedNs,
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{}, // if this file ever resurfaces anywhere we want our delete to be strictly older
}
} else {
// Revert means to throw away our local changes. We reset the
// version to the empty vector, which is strictly older than any
// other existing version. It is not in conflict with anything,
// either, so we will not create a conflict copy of our local
// changes.
fi.Version = protocol.Vector{}
fi.LocalFlags &^= protocol.FlagLocalReceiveOnly
}
batch = append(batch, fi)
batchSizeBytes += fi.ProtoSize()
if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes {
updateFn(batch)
batch = batch[:0]
batchSizeBytes = 0
}
return true
})
if len(batch) > 0 {
updateFn(batch)
}
batch = batch[:0]
batchSizeBytes = 0
// Handle any queued directories
deleted, err := delQueue.flush()
if err != nil {
l.Infoln("Revert:", err)
}
now := time.Now()
for _, dir := range deleted {
batch = append(batch, protocol.FileInfo{
Name: dir,
Type: protocol.FileInfoTypeDirectory,
ModifiedS: now.Unix(),
ModifiedBy: f.shortID,
Deleted: true,
Version: protocol.Vector{},
})
}
if len(batch) > 0 {
updateFn(batch)
}
// We will likely have changed our local index, but that won't trigger a
// pull by itself. Make sure we schedule one so that we start
// downloading files.
f.SchedulePull()
}
// deleteQueue handles deletes by delegating to a handler and queuing
// directories for last.
type deleteQueue struct {
handler interface {
deleteFile(file protocol.FileInfo) (dbUpdateJob, error)
deleteDir(dir string, ignores *ignore.Matcher, scanChan chan<- string) error
}
ignores *ignore.Matcher
dirs []string
}
func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) {
// Things that are ignored but not marked deletable are not processed.
ign := q.ignores.Match(fi.Name)
if ign.IsIgnored() && !ign.IsDeletable() {
return false, nil
}
// Directories are queued for later processing.
if fi.IsDirectory() {
q.dirs = append(q.dirs, fi.Name)
return false, nil
}
// Kill it.
_, err := q.handler.deleteFile(fi)
return true, err
}
func (q *deleteQueue) flush() ([]string, error) {
// Process directories from the leaves inward.
sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))
var firstError error
var deleted []string
for _, dir := range q.dirs {
if err := q.handler.deleteDir(dir, q.ignores, nil); err == nil {
deleted = append(deleted, dir)
} else if err != nil && firstError == nil {
firstError = err
}
}
return deleted, firstError
}

261
lib/model/folder_recvonly_test.go Normal file
View File

@ -0,0 +1,261 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package model
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
)
func TestRecvOnlyRevertDeletes(t *testing.T) {
// Make sure that we delete extraneous files and directories when we hit
// Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
os.MkdirAll("_recvonly/ignDir", 0755)
os.MkdirAll("_recvonly/unknownDir", 0755)
ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644)
ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644)
knownFiles := setupKnownFiles(t, []byte("hello\n"))
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
// Start the folder. This will cause a scan, should discover the other stuff in the folder
m.StartFolder("ro")
m.ScanFolder("ro")
// We should now have two files and two directories.
size = m.GlobalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Global: expected 2 files and 2 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 2 || size.Directories != 2 {
t.Fatalf("Local: expected 2 files and 2 directories: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// Revert should delete the unknown stuff
m.Revert("ro")
// These should still exist
for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} {
_, err := os.Stat(p)
if err != nil {
t.Error("Unexpected error:", err)
}
}
// These should have been removed
for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} {
_, err := os.Stat(p)
if !os.IsNotExist(err) {
t.Error("Unexpected existing thing:", p)
}
}
// We should now have one file and directory again.
size = m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 files and 1 directories: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 files and 1 directories: %+v", size)
}
}
func TestRecvOnlyRevertNeeds(t *testing.T) {
// Make sure that a new file gets picked up and considered latest, then
// gets considered old when we hit Revert.
os.RemoveAll("_recvonly")
defer os.RemoveAll("_recvonly")
// Create some test data
os.MkdirAll("_recvonly/.stfolder", 0755)
oldData := []byte("hello\n")
knownFiles := setupKnownFiles(t, oldData)
// Get us a model up and running
m := setupROFolder()
defer m.Stop()
// Send and index update for the known stuff
m.Index(device1, "ro", knownFiles)
m.updateLocalsFromScanning("ro", knownFiles)
// Start the folder. This will cause a scan.
m.StartFolder("ro")
m.ScanFolder("ro")
// Everything should be in sync.
size := m.GlobalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Global: expected 1 file and 1 directory: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Directories != 1 {
t.Fatalf("Local: expected 1 file and 1 directory: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("ROChanged: expected nothing: %+v", size)
}
// Update the file.
newData := []byte("totally different data\n")
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644); err != nil {
t.Fatal(err)
}
// Rescan.
if err := m.ScanFolder("ro"); err != nil {
t.Fatal(err)
}
// We now have a newer file than the rest of the cluster. Global state should reflect this.
size = m.GlobalSize("ro")
const sizeOfDir = 128
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Global: expected the new file to be reflected: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the new file to be reflected: %+v", size)
}
size = m.NeedSize("ro")
if size.Files+size.Directories > 0 {
t.Fatalf("Need: expected nothing: %+v", size)
}
size = m.ReceiveOnlyChangedSize("ro")
if size.Files+size.Directories == 0 {
t.Fatalf("ROChanged: expected something: %+v", size)
}
// We hit the Revert button. The file that was new should become old.
m.Revert("ro")
size = m.GlobalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) {
t.Fatalf("Global: expected the global size to revert: %+v", size)
}
size = m.LocalSize("ro")
if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) {
t.Fatalf("Local: expected the local size to remain: %+v", size)
}
size = m.NeedSize("ro")
if size.Files != 1 || size.Bytes != int64(len(oldData)) {
t.Fatalf("Local: expected to need the old file data: %+v", size)
}
}
func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo {
if err := os.MkdirAll("_recvonly/knownDir", 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644); err != nil {
t.Fatal(err)
}
t0 := time.Now().Add(-1 * time.Minute)
if err := os.Chtimes("_recvonly/knownDir/knownFile", t0, t0); err != nil {
t.Fatal(err)
}
fi, err := os.Stat("_recvonly/knownDir/knownFile")
if err != nil {
t.Fatal(err)
}
knownFiles := []protocol.FileInfo{
{
Name: "knownDir",
Type: protocol.FileInfoTypeDirectory,
Permissions: 0755,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
{
Name: "knownDir/knownFile",
Type: protocol.FileInfoTypeFile,
Permissions: 0644,
Size: fi.Size(),
ModifiedS: fi.ModTime().Unix(),
ModifiedNs: int32(fi.ModTime().UnixNano() % 1e9),
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}},
Sequence: 42,
},
}
return knownFiles
}
func setupROFolder() *Model {
fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly")
fcfg.Type = config.FolderTypeReceiveOnly
fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}}
cfg := defaultCfg.Copy()
cfg.Folders = append(cfg.Folders, fcfg)
wrp := config.Wrap("/dev/null", cfg)
db := db.OpenMemory()
m := NewModel(wrp, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
m.ServeBackground()
m.AddFolder(fcfg)
return m
}

2
lib/model/folder_sendonly.go
View File

@ -76,7 +76,7 @@ func (f *sendOnlyFolder) pull() bool {
} }
file := intf.(protocol.FileInfo) file := intf.(protocol.FileInfo)
if !file.IsEquivalent(curFile, f.IgnorePerms, false) { if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) {
return true return true
} }

32
lib/model/folder_sendrecv.go
View File

@ -501,7 +501,11 @@ func (f *sendReceiveFolder) processDeletions(ignores *ignore.Matcher, fileDeleti
} }
l.Debugln(f, "Deleting file", file.Name) l.Debugln(f, "Deleting file", file.Name)
f.deleteFile(file, dbUpdateChan) if update, err := f.deleteFile(file); err != nil {
f.newError("delete file", file.Name, err)
} else {
dbUpdateChan <- update
}
} }
for i := range dirDeletions { for i := range dirDeletions {
@ -736,7 +740,7 @@ func (f *sendReceiveFolder) handleDeleteDir(file protocol.FileInfo, ignores *ign
} }
// deleteFile attempts to delete the given file // deleteFile attempts to delete the given file
func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) { func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) (dbUpdateJob, error) {
// Used in the defer closure below, updated by the function body. Take // Used in the defer closure below, updated by the function body. Take
// care not declare another err. // care not declare another err.
var err error var err error
@ -775,16 +779,18 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan
if err == nil || fs.IsNotExist(err) { if err == nil || fs.IsNotExist(err) {
// It was removed or it doesn't exist to start with // It was removed or it doesn't exist to start with
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} return dbUpdateJob{file, dbUpdateDeleteFile}, nil
} else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { }
if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) {
// We get an error just looking at the file, and it's not a permission // We get an error just looking at the file, and it's not a permission
// problem. Lets assume the error is in fact some variant of "file // problem. Lets assume the error is in fact some variant of "file
// does not exist" (possibly expressed as some parent being a file and // does not exist" (possibly expressed as some parent being a file and
// not a directory etc) and that the delete is handled. // not a directory etc) and that the delete is handled.
dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} return dbUpdateJob{file, dbUpdateDeleteFile}, nil
} else {
f.newError("delete file", file.Name, err)
} }
return dbUpdateJob{}, err
} }
// renameFile attempts to rename an existing file to a destination // renameFile attempts to rename an existing file to a destination
@ -1778,10 +1784,14 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC
} else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() { } else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() {
hasIgnored = true hasIgnored = true
} else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() { } else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() {
// Something appeared in the dir that we either are not // Something appeared in the dir that we either are not aware of
// aware of at all, that we think should be deleted or that // at all, that we think should be deleted or that is invalid,
// is invalid, but not currently ignored -> schedule scan // but not currently ignored -> schedule scan. The scanChan
scanChan <- fullDirFile // might be nil, in which case we trust the scanning to be
// handled later as a result of our error return.
if scanChan != nil {
scanChan <- fullDirFile
}
hasToBeScanned = true hasToBeScanned = true
} else { } else {
// Dir contains file that is valid according to db and // Dir contains file that is valid according to db and

4
lib/model/folder_sendrecv_test.go
View File

@ -74,12 +74,12 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
} }
} }
func setUpModel(file protocol.FileInfo) *Model { func setUpModel(files ...protocol.FileInfo) *Model {
db := db.OpenMemory() db := db.OpenMemory()
model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil) model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil)
model.AddFolder(defaultFolderConfig) model.AddFolder(defaultFolderConfig)
// Update index // Update index
model.updateLocalsFromScanning("default", []protocol.FileInfo{file}) model.updateLocalsFromScanning("default", files)
return model return model
} }

41
lib/model/model.go
View File

@ -57,6 +57,7 @@ const (
type service interface { type service interface {
BringToFront(string) BringToFront(string)
Override(*db.FileSet, func([]protocol.FileInfo)) Override(*db.FileSet, func([]protocol.FileInfo))
Revert(*db.FileSet, func([]protocol.FileInfo))
DelayScan(d time.Duration) DelayScan(d time.Duration)
IgnoresUpdated() // ignore matcher was updated notification IgnoresUpdated() // ignore matcher was updated notification
SchedulePull() // something relevant changed, we should try a pull SchedulePull() // something relevant changed, we should try a pull
@ -690,6 +691,18 @@ func (m *Model) LocalSize(folder string) db.Counts {
return db.Counts{} return db.Counts{}
} }
// ReceiveOnlyChangedSize returns the number of files, deleted files and
// total bytes for all files that have changed locally in a receieve only
// folder.
func (m *Model) ReceiveOnlyChangedSize(folder string) db.Counts {
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
return rf.ReceiveOnlyChangedSize()
}
return db.Counts{}
}
// NeedSize returns the number and total size of currently needed files. // NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) db.Counts { func (m *Model) NeedSize(folder string) db.Counts {
m.fmut.RLock() m.fmut.RLock()
@ -1747,6 +1760,12 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs
// Mark the file as invalid if any of the local bad stuff flags are set. // Mark the file as invalid if any of the local bad stuff flags are set.
f.RawInvalid = f.IsInvalid() f.RawInvalid = f.IsInvalid()
// If the file is marked LocalReceive (i.e., changed locally on a
// receive only folder) we do not want it to ever become the
// globally best version, invalid or not.
if f.IsReceiveOnlyChanged() {
f.Version = protocol.Vector{}
}
f.LocalFlags = 0 // never sent externally f.LocalFlags = 0 // never sent externally
if dropSymlinks && f.IsSymlink() { if dropSymlinks && f.IsSymlink() {
@ -1940,7 +1959,7 @@ func (m *Model) ScanFolderSubdirs(folder string, subs []string) error {
return runner.Scan(subs) return runner.Scan(subs)
} }
func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string) error { func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string, localFlags uint32) error {
m.fmut.RLock() m.fmut.RLock()
if err := m.checkFolderRunningLocked(folder); err != nil { if err := m.checkFolderRunningLocked(folder); err != nil {
m.fmut.RUnlock() m.fmut.RUnlock()
@ -2010,6 +2029,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ShortID: m.shortID, ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS, ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseLargeBlocks: folderCfg.UseLargeBlocks, UseLargeBlocks: folderCfg.UseLargeBlocks,
LocalFlags: localFlags,
}) })
if err := runner.CheckHealth(); err != nil { if err := runner.CheckHealth(); err != nil {
@ -2106,6 +2126,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ModifiedBy: m.id.Short(), ModifiedBy: m.id.Short(),
Deleted: true, Deleted: true,
Version: f.Version.Update(m.shortID), Version: f.Version.Update(m.shortID),
LocalFlags: localFlags,
} }
// We do not want to override the global version // We do not want to override the global version
// with the deleted file. Keeping only our local // with the deleted file. Keeping only our local
@ -2289,6 +2310,24 @@ func (m *Model) Override(folder string) {
}) })
} }
func (m *Model) Revert(folder string) {
// Grab the runner and the file set.
m.fmut.RLock()
fs, fsOK := m.folderFiles[folder]
runner, runnerOK := m.folderRunners[folder]
m.fmut.RUnlock()
if !fsOK || !runnerOK {
return
}
// Run the revert, taking updates as if they came from scanning.
runner.Revert(fs, func(files []protocol.FileInfo) {
m.updateLocalsFromScanning(folder, files)
})
}
// CurrentSequence returns the change version for the given folder. // CurrentSequence returns the change version for the given folder.
// This is guaranteed to increment if the contents of the local folder has // This is guaranteed to increment if the contents of the local folder has
// changed. // changed.

30
lib/protocol/bep_extensions.go
View File

@ -49,6 +49,10 @@ func (f FileInfo) IsInvalid() bool {
return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0 return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0
} }
func (f FileInfo) IsUnsupported() bool {
return f.LocalFlags&FlagLocalUnsupported != 0
}
func (f FileInfo) IsIgnored() bool { func (f FileInfo) IsIgnored() bool {
return f.LocalFlags&FlagLocalIgnored != 0 return f.LocalFlags&FlagLocalIgnored != 0
} }
@ -57,6 +61,10 @@ func (f FileInfo) MustRescan() bool {
return f.LocalFlags&FlagLocalMustRescan != 0 return f.LocalFlags&FlagLocalMustRescan != 0
} }
func (f FileInfo) IsReceiveOnlyChanged() bool {
return f.LocalFlags&FlagLocalReceiveOnly != 0
}
func (f FileInfo) IsDirectory() bool { func (f FileInfo) IsDirectory() bool {
return f.Type == FileInfoTypeDirectory return f.Type == FileInfoTypeDirectory
} }
@ -99,6 +107,10 @@ func (f FileInfo) FileName() string {
return f.Name return f.Name
} }
func (f FileInfo) FileLocalFlags() uint32 {
return f.LocalFlags
}
func (f FileInfo) ModTime() time.Time { func (f FileInfo) ModTime() time.Time {
return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) return time.Unix(f.ModifiedS, int64(f.ModifiedNs))
} }
@ -114,7 +126,7 @@ func (f FileInfo) FileVersion() Vector {
// WinsConflict returns true if "f" is the one to choose when it is in // WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other". // conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool { func (f FileInfo) WinsConflict(other FileInfo) bool {
// If only one of the files is invalid, that one loses // If only one of the files is invalid, that one loses.
if f.IsInvalid() != other.IsInvalid() { if f.IsInvalid() != other.IsInvalid() {
return !f.IsInvalid() return !f.IsInvalid()
} }
@ -145,7 +157,15 @@ func (f FileInfo) IsEmpty() bool {
return f.Version.Counters == nil return f.Version.Counters == nil
} }
// IsEquivalent checks that the two file infos represent the same actual file content, func (f FileInfo) IsEquivalent(other FileInfo) bool {
return f.isEquivalent(other, false, false, 0)
}
func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags)
}
// isEquivalent checks that the two file infos represent the same actual file content,
// i.e. it does purposely not check only selected (see below) struct members. // i.e. it does purposely not check only selected (see below) struct members.
// Permissions (config) and blocks (scanning) can be excluded from the comparison. // Permissions (config) and blocks (scanning) can be excluded from the comparison.
// Any file info is not "equivalent", if it has different // Any file info is not "equivalent", if it has different
@ -160,7 +180,7 @@ func (f FileInfo) IsEmpty() bool {
// A symlink is not "equivalent", if it has different // A symlink is not "equivalent", if it has different
// - target // - target
// A directory does not have anything specific to check. // A directory does not have anything specific to check.
func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool) bool { func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool {
if f.MustRescan() || other.MustRescan() { if f.MustRescan() || other.MustRescan() {
// These are per definition not equivalent because they don't // These are per definition not equivalent because they don't
// represent a valid state, even if both happen to have the // represent a valid state, even if both happen to have the
@ -168,6 +188,10 @@ func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo
return false return false
} }
// Mask out the ignored local flags before checking IsInvalid() below
f.LocalFlags &^= ignoreFlags
other.LocalFlags &^= ignoreFlags
if f.Name != other.Name || f.Type != other.Type || f.Deleted != other.Deleted || f.IsInvalid() != other.IsInvalid() { if f.Name != other.Name || f.Type != other.Type || f.Deleted != other.Deleted || f.IsInvalid() != other.IsInvalid() {
return false return false
} }

12
lib/protocol/deviceid.go
View File

@ -19,10 +19,18 @@ type DeviceID [DeviceIDLength]byte
type ShortID uint64 type ShortID uint64
var ( var (
LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} LocalDeviceID = repeatedDeviceID(0xff)
EmptyDeviceID = DeviceID{ /* all zeroes */ } GlobalDeviceID = repeatedDeviceID(0xf8)
EmptyDeviceID = DeviceID{ /* all zeroes */ }
) )
func repeatedDeviceID(v byte) (d DeviceID) {
for i := range d {
d[i] = v
}
return
}
// NewDeviceID generates a new device ID from the raw bytes of a certificate // NewDeviceID generates a new device ID from the raw bytes of a certificate
func NewDeviceID(rawCert []byte) DeviceID { func NewDeviceID(rawCert []byte) DeviceID {
var n DeviceID var n DeviceID

5
lib/protocol/protocol.go
View File

@ -94,14 +94,15 @@ const (
FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows
FlagLocalIgnored = 1 << 1 // Matches local ignore patterns FlagLocalIgnored = 1 << 1 // Matches local ignore patterns
FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully
FlagLocalReceiveOnly = 1 << 3 // Change detected on receive only folder
// Flags that should result in the Invalid bit on outgoing updates // Flags that should result in the Invalid bit on outgoing updates
LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly
// Flags that should result in a file being in conflict with its // Flags that should result in a file being in conflict with its
// successor, due to us not having an up to date picture of its state on // successor, due to us not having an up to date picture of its state on
// disk. // disk.
LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly
) )
var ( var (

16
lib/protocol/protocol_test.go
View File

@ -423,6 +423,7 @@ func TestIsEquivalent(t *testing.T) {
b FileInfo b FileInfo
ignPerms *bool // nil means should not matter, we'll test both variants ignPerms *bool // nil means should not matter, we'll test both variants
ignBlocks *bool ignBlocks *bool
ignFlags uint32
eq bool eq bool
} }
cases := []testCase{ cases := []testCase{
@ -491,6 +492,17 @@ func TestIsEquivalent(t *testing.T) {
b: FileInfo{LocalFlags: FlagLocalUnsupported}, b: FileInfo{LocalFlags: FlagLocalUnsupported},
eq: true, eq: true,
}, },
{
a: FileInfo{LocalFlags: 0},
b: FileInfo{LocalFlags: FlagLocalReceiveOnly},
eq: false,
},
{
a: FileInfo{LocalFlags: 0},
b: FileInfo{LocalFlags: FlagLocalReceiveOnly},
ignFlags: FlagLocalReceiveOnly,
eq: true,
},
// Difference in blocks is not OK // Difference in blocks is not OK
{ {
@ -588,10 +600,10 @@ func TestIsEquivalent(t *testing.T) {
continue continue
} }
if res := tc.a.IsEquivalent(tc.b, ignPerms, ignBlocks); res != tc.eq { if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
} }
if res := tc.b.IsEquivalent(tc.a, ignPerms, ignBlocks); res != tc.eq { if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq {
t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq)
} }
} }

11
lib/scanner/walk.go
View File

@ -68,6 +68,8 @@ type Config struct {
ProgressTickIntervalS int ProgressTickIntervalS int
// Whether to use large blocks for large files or the old standard of 128KiB for everything. // Whether to use large blocks for large files or the old standard of 128KiB for everything.
UseLargeBlocks bool UseLargeBlocks bool
// Local flags to set on scanned files
LocalFlags uint32
} }
type CurrentFiler interface { type CurrentFiler interface {
@ -367,10 +369,11 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
ModifiedBy: w.ShortID, ModifiedBy: w.ShortID,
Size: info.Size(), Size: info.Size(),
RawBlockSize: int32(blockSize), RawBlockSize: int32(blockSize),
LocalFlags: w.LocalFlags,
} }
if hasCurFile { if hasCurFile {
if curFile.IsEquivalent(f, w.IgnorePerms, true) { if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil return nil
} }
if curFile.ShouldConflict() { if curFile.ShouldConflict() {
@ -407,10 +410,11 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo,
ModifiedS: info.ModTime().Unix(), ModifiedS: info.ModTime().Unix(),
ModifiedNs: int32(info.ModTime().Nanosecond()), ModifiedNs: int32(info.ModTime().Nanosecond()),
ModifiedBy: w.ShortID, ModifiedBy: w.ShortID,
LocalFlags: w.LocalFlags,
} }
if ok { if ok {
if cf.IsEquivalent(f, w.IgnorePerms, true) { if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil return nil
} }
if cf.ShouldConflict() { if cf.ShouldConflict() {
@ -463,10 +467,11 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan pro
NoPermissions: true, // Symlinks don't have permissions of their own NoPermissions: true, // Symlinks don't have permissions of their own
SymlinkTarget: target, SymlinkTarget: target,
ModifiedBy: w.ShortID, ModifiedBy: w.ShortID,
LocalFlags: w.LocalFlags,
} }
if ok { if ok {
if cf.IsEquivalent(f, w.IgnorePerms, true) { if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) {
return nil return nil
} }
if cf.ShouldConflict() { if cf.ShouldConflict() {

69
lib/scanner/walk_test.go
View File

@ -221,8 +221,8 @@ func TestNormalization(t *testing.T) {
// make sure it all gets done. In production, things will be correct // make sure it all gets done. In production, things will be correct
// eventually... // eventually...
walkDir(testFs, "normalization", nil, nil) walkDir(testFs, "normalization", nil, nil, 0)
tmp := walkDir(testFs, "normalization", nil, nil) tmp := walkDir(testFs, "normalization", nil, nil, 0)
files := fileList(tmp).testfiles() files := fileList(tmp).testfiles()
@ -267,7 +267,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks") fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
for _, path := range []string{".", "link"} { for _, path := range []string{".", "link"} {
// Scan it // Scan it
files := walkDir(fs, path, nil, nil) files := walkDir(fs, path, nil, nil, 0)
// Verify that we got one symlink and with the correct attributes // Verify that we got one symlink and with the correct attributes
if len(files) != 1 { if len(files) != 1 {
@ -300,7 +300,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
for _, path := range []string{".", "link"} { for _, path := range []string{".", "link"} {
// Scan it // Scan it
files := walkDir(fs, path, nil, nil) files := walkDir(fs, path, nil, nil, 0)
// Verify that we got zero symlinks // Verify that we got zero symlinks
if len(files) != 0 { if len(files) != 0 {
@ -329,7 +329,7 @@ func TestWalkRootSymlink(t *testing.T) {
} }
// Scan it // Scan it
files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil) files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
// Verify that we got two files // Verify that we got two files
if len(files) != 2 { if len(files) != 2 {
@ -353,7 +353,7 @@ func TestBlocksizeHysteresis(t *testing.T) {
current := make(fakeCurrentFiler) current := make(fakeCurrentFiler)
runTest := func(expectedBlockSize int) { runTest := func(expectedBlockSize int) {
files := walkDir(sf, ".", current, nil) files := walkDir(sf, ".", current, nil, 0)
if len(files) != 1 { if len(files) != 1 {
t.Fatalf("expected one file, not %d", len(files)) t.Fatalf("expected one file, not %d", len(files))
} }
@ -407,7 +407,57 @@ func TestBlocksizeHysteresis(t *testing.T) {
runTest(512 << 10) runTest(512 << 10)
} }
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher) []protocol.FileInfo { func TestWalkReceiveOnly(t *testing.T) {
sf := fs.NewWalkFilesystem(&singleFileFS{
name: "testfile.dat",
filesize: 1024,
})
current := make(fakeCurrentFiler)
// Initial scan, no files in the CurrentFiler. Should pick up the file and
// set the ReceiveOnly flag on it, because that's the flag we give the
// walker to set.
files := walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 1 {
t.Fatal("Should have scanned one file")
}
if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
t.Fatal("Should have set the ReceiveOnly flag")
}
// Update the CurrentFiler and scan again. It should not return
// anything, because the file has not changed. This verifies that the
// ReceiveOnly flag is properly ignored and doesn't trigger a rescan
// every time.
cur := files[0]
current[cur.Name] = cur
files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 0 {
t.Fatal("Should not have scanned anything")
}
// Now pretend the file was previously ignored instead. We should pick up
// the difference in flags and set just the LocalReceive flags.
cur.LocalFlags = protocol.FlagLocalIgnored
current[cur.Name] = cur
files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
if len(files) != 1 {
t.Fatal("Should have scanned one file")
}
if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
t.Fatal("Should have set the ReceiveOnly flag")
}
}
func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo {
fchan := Walk(context.TODO(), Config{ fchan := Walk(context.TODO(), Config{
Filesystem: fs, Filesystem: fs,
Subs: []string{dir}, Subs: []string{dir},
@ -416,6 +466,7 @@ func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.
UseLargeBlocks: true, UseLargeBlocks: true,
CurrentFiler: cfiler, CurrentFiler: cfiler,
Matcher: matcher, Matcher: matcher,
LocalFlags: localFlags,
}) })
var tmp []protocol.FileInfo var tmp []protocol.FileInfo
@ -579,7 +630,7 @@ func TestIssue4799(t *testing.T) {
} }
fd.Close() fd.Close()
files := walkDir(fs, "/foo", nil, nil) files := walkDir(fs, "/foo", nil, nil, 0)
if len(files) != 1 || files[0].Name != "foo" { if len(files) != 1 || files[0].Name != "foo" {
t.Error(`Received unexpected file infos when walking "/foo"`, files) t.Error(`Received unexpected file infos when walking "/foo"`, files)
} }
@ -597,7 +648,7 @@ func TestRecurseInclude(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
files := walkDir(testFs, ".", nil, ignores) files := walkDir(testFs, ".", nil, ignores, 0)
expected := []string{ expected := []string{
filepath.Join("dir1"), filepath.Join("dir1"),