diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 90f0b2e78..e57db76db 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -85,6 +85,7 @@ type modelIntf interface { GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{} Completion(device protocol.DeviceID, folder string) model.FolderCompletion Override(folder string) + Revert(folder string) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) NeedSize(folder string) db.Counts @@ -107,6 +108,7 @@ type modelIntf interface { Connection(deviceID protocol.DeviceID) (connections.Connection, bool) GlobalSize(folder string) db.Counts LocalSize(folder string) db.Counts + ReceiveOnlyChangedSize(folder string) db.Counts CurrentSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool) State(folder string) (string, time.Time, error) @@ -293,6 +295,7 @@ func (s *apiService) Serve() { postRestMux.HandleFunc("/rest/db/prio", s.postDBPrio) // folder file [perpage] [page] postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores) // folder postRestMux.HandleFunc("/rest/db/override", s.postDBOverride) // folder + postRestMux.HandleFunc("/rest/db/revert", s.postDBRevert) // folder postRestMux.HandleFunc("/rest/db/scan", s.postDBScan) // folder [sub...] [delay] postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore) // folder postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig) // @@ -712,6 +715,17 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter need := m.NeedSize(folder) res["needFiles"], res["needDirectories"], res["needSymlinks"], res["needDeletes"], res["needBytes"] = need.Files, need.Directories, need.Symlinks, need.Deleted, need.Bytes + if cfg.Folders()[folder].Type == config.FolderTypeReceiveOnly { + // Add statistics for things that have changed locally in a receive + // only folder. + ro := m.ReceiveOnlyChangedSize(folder) + res["receiveOnlyChangedFiles"] = ro.Files + res["receiveOnlyChangedDirectories"] = ro.Directories + res["receiveOnlyChangedSymlinks"] = ro.Symlinks + res["receiveOnlyChangedDeletes"] = ro.Deleted + res["receiveOnlyChangedBytes"] = ro.Bytes + } + res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes res["state"], res["stateChanged"], err = m.State(folder) @@ -748,6 +762,12 @@ func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) { go s.model.Override(folder) } +func (s *apiService) postDBRevert(w http.ResponseWriter, r *http.Request) { + var qs = r.URL.Query() + var folder = qs.Get("folder") + go s.model.Revert(folder) +} + func getPagingParams(qs url.Values) (int, int) { page, err := strconv.Atoi(qs.Get("page")) if err != nil || page < 1 { diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 5841b99a0..04e854aa1 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -29,6 +29,8 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model. func (m *mockedModel) Override(folder string) {} +func (m *mockedModel) Revert(folder string) {} + func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) { return nil, nil, nil } @@ -117,6 +119,10 @@ func (m *mockedModel) LocalSize(folder string) db.Counts { return db.Counts{} } +func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts { + return db.Counts{} +} + func (m *mockedModel) CurrentSequence(folder string) (int64, bool) { return 0, false } diff --git a/gui/default/index.html b/gui/default/index.html index 7dd6786ae..c2a10c0eb 100644 --- a/gui/default/index.html +++ b/gui/default/index.html @@ -301,7 +301,9 @@

◼ @@ -386,9 +388,10 @@ -  Folder Type +  Folder Type Send Only + Receive Only @@ -478,6 +481,9 @@ +
diff --git a/lib/config/foldertype.go b/lib/config/foldertype.go index 1c82bb42d..e76ef0884 100644 --- a/lib/config/foldertype.go +++ b/lib/config/foldertype.go @@ -11,6 +11,7 @@ type FolderType int const ( FolderTypeSendReceive FolderType = iota // default is sendreceive FolderTypeSendOnly + FolderTypeReceiveOnly ) func (t FolderType) String() string { @@ -19,6 +20,8 @@ func (t FolderType) String() string { return "sendreceive" case FolderTypeSendOnly: return "sendonly" + case FolderTypeReceiveOnly: + return "receiveonly" default: return "unknown" } @@ -34,6 +37,8 @@ func (t *FolderType) UnmarshalText(bs []byte) error { *t = FolderTypeSendReceive case "readonly", "sendonly": *t = FolderTypeSendOnly + case "receiveonly": + *t = FolderTypeReceiveOnly default: *t = FolderTypeSendReceive } diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index 87b13afa5..7d67ea671 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -586,7 +586,7 @@ func (db *Instance) checkGlobals(folder []byte, meta *metadataTracker) { if i == 0 { if fi, ok := db.getFile(fk); ok { - meta.addFile(globalDeviceID, fi) + meta.addFile(protocol.GlobalDeviceID, fi) } } } diff --git a/lib/db/leveldb_test.go b/lib/db/leveldb_test.go index 85196259e..dcdb5f41a 100644 --- a/lib/db/leveldb_test.go +++ b/lib/db/leveldb_test.go @@ -305,7 +305,7 @@ func TestUpdate0to3(t *testing.T) { t.Error("Unexpected additional file via sequence", f.FileName()) return true } - if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) { + if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) { found = true } else { t.Errorf("Wrong file via sequence, got %v, expected %v", f, e) @@ -330,7 +330,7 @@ func TestUpdate0to3(t *testing.T) { } f := fi.(protocol.FileInfo) delete(need, f.Name) - if !f.IsEquivalent(e, true, true) { + if !f.IsEquivalentOptional(e, true, true, 0) { t.Errorf("Wrong needed file, got %v, expected %v", f, e) } return true diff --git a/lib/db/leveldb_transactions.go b/lib/db/leveldb_transactions.go index a9ce5ddb7..64496380f 100644 --- a/lib/db/leveldb_transactions.go +++ b/lib/db/leveldb_transactions.go @@ -140,11 +140,11 @@ func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file proto if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok { // A failure to get the file here is surprising and our // global size data will be incorrect until a restart... - meta.removeFile(globalDeviceID, oldFile) + meta.removeFile(protocol.GlobalDeviceID, oldFile) } // Add the new global to the global size counter - meta.addFile(globalDeviceID, newGlobal) + meta.addFile(protocol.GlobalDeviceID, newGlobal) l.Debugf(`new global for "%v" after update: %v`, file.Name, fl) t.Put(gk, mustMarshal(&fl)) @@ -197,7 +197,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, // didn't exist anyway, apparently continue } - meta.removeFile(globalDeviceID, f) + meta.removeFile(protocol.GlobalDeviceID, f) removed = true } fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...) @@ -215,7 +215,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte, if f, ok := t.getFile(folder, fl.Versions[0].Device, file); ok { // A failure to get the file here is surprising and our // global size data will be incorrect until a restart... - meta.addFile(globalDeviceID, f) + meta.addFile(protocol.GlobalDeviceID, f) } } } diff --git a/lib/db/meta.go b/lib/db/meta.go index 09015b174..93dc2a59d 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -7,25 +7,30 @@ package db import ( + "bytes" + "math/bits" "time" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) -// like protocol.LocalDeviceID but with 0xf8 in all positions -var globalDeviceID = protocol.DeviceID{0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8} - +// metadataTracker keeps metadata on a per device, per local flag basis. type metadataTracker struct { mut sync.RWMutex counts CountsSet - indexes map[protocol.DeviceID]int // device ID -> index in counts + indexes map[metaKey]int // device ID + local flags -> index in counts +} + +type metaKey struct { + dev protocol.DeviceID + flags uint32 } func newMetadataTracker() *metadataTracker { return &metadataTracker{ mut: sync.NewRWMutex(), - indexes: make(map[protocol.DeviceID]int), + indexes: make(map[metaKey]int), } } @@ -38,7 +43,7 @@ func (m *metadataTracker) Unmarshal(bs []byte) error { // Initialize the index map for i, c := range m.counts.Counts { - m.indexes[protocol.DeviceIDFromBytes(c.DeviceID)] = i + m.indexes[metaKey{protocol.DeviceIDFromBytes(c.DeviceID), c.LocalFlags}] = i } return nil } @@ -72,14 +77,15 @@ func (m *metadataTracker) fromDB(db *Instance, folder []byte) error { // countsPtr returns a pointer to the corresponding Counts struct, if // necessary allocating one in the process -func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts { +func (m *metadataTracker) countsPtr(dev protocol.DeviceID, flags uint32) *Counts { // must be called with the mutex held - idx, ok := m.indexes[dev] + key := metaKey{dev, flags} + idx, ok := m.indexes[key] if !ok { idx = len(m.counts.Counts) - m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:]}) - m.indexes[dev] = idx + m.counts.Counts = append(m.counts.Counts, Counts{DeviceID: dev[:], LocalFlags: flags}) + m.indexes[key] = idx } return &m.counts.Counts[idx] } @@ -87,12 +93,23 @@ func (m *metadataTracker) countsPtr(dev protocol.DeviceID) *Counts { // addFile adds a file to the counts, adjusting the sequence number as // appropriate func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { - if f.IsInvalid() { - return + m.mut.Lock() + + if flags := f.FileLocalFlags(); flags == 0 { + // Account regular files in the zero-flags bucket. + m.addFileLocked(dev, 0, f) + } else { + // Account in flag specific buckets. + eachFlagBit(flags, func(flag uint32) { + m.addFileLocked(dev, flag, f) + }) } - m.mut.Lock() - cp := m.countsPtr(dev) + m.mut.Unlock() +} + +func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) { + cp := m.countsPtr(dev, flags) switch { case f.IsDeleted(): @@ -109,18 +126,27 @@ func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { if seq := f.SequenceNo(); seq > cp.Sequence { cp.Sequence = seq } - - m.mut.Unlock() } // removeFile removes a file from the counts func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { - if f.IsInvalid() { - return + m.mut.Lock() + + if flags := f.FileLocalFlags(); flags == 0 { + // Remove regular files from the zero-flags bucket + m.removeFileLocked(dev, 0, f) + } else { + // Remove from flag specific buckets. + eachFlagBit(flags, func(flag uint32) { + m.removeFileLocked(dev, flag, f) + }) } - m.mut.Lock() - cp := m.countsPtr(dev) + m.mut.Unlock() +} + +func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flags uint32, f FileIntf) { + cp := m.countsPtr(dev, f.FileLocalFlags()) switch { case f.IsDeleted(): @@ -153,14 +179,19 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { cp.Symlinks = 0 m.counts.Created = 0 } - - m.mut.Unlock() } // resetAll resets all metadata for the given device func (m *metadataTracker) resetAll(dev protocol.DeviceID) { m.mut.Lock() - *m.countsPtr(dev) = Counts{DeviceID: dev[:]} + for i, c := range m.counts.Counts { + if bytes.Equal(c.DeviceID, dev[:]) { + m.counts.Counts[i] = Counts{ + DeviceID: c.DeviceID, + LocalFlags: c.LocalFlags, + } + } + } m.mut.Unlock() } @@ -169,23 +200,30 @@ func (m *metadataTracker) resetAll(dev protocol.DeviceID) { func (m *metadataTracker) resetCounts(dev protocol.DeviceID) { m.mut.Lock() - c := m.countsPtr(dev) - c.Bytes = 0 - c.Deleted = 0 - c.Directories = 0 - c.Files = 0 - c.Symlinks = 0 - // c.Sequence deliberately untouched + for i, c := range m.counts.Counts { + if bytes.Equal(c.DeviceID, dev[:]) { + m.counts.Counts[i] = Counts{ + DeviceID: c.DeviceID, + Sequence: c.Sequence, + LocalFlags: c.LocalFlags, + } + } + } m.mut.Unlock() } -// Counts returns the counts for the given device ID -func (m *metadataTracker) Counts(dev protocol.DeviceID) Counts { +// Counts returns the counts for the given device ID and flag. `flag` should +// be zero or have exactly one bit set. +func (m *metadataTracker) Counts(dev protocol.DeviceID, flag uint32) Counts { + if bits.OnesCount32(flag) > 1 { + panic("incorrect usage: set at most one bit in flag") + } + m.mut.RLock() defer m.mut.RUnlock() - idx, ok := m.indexes[dev] + idx, ok := m.indexes[metaKey{dev, flag}] if !ok { return Counts{} } @@ -198,7 +236,7 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 { m.mut.Lock() defer m.mut.Unlock() - c := m.countsPtr(dev) + c := m.countsPtr(dev, 0) c.Sequence++ return c.Sequence } @@ -206,21 +244,26 @@ func (m *metadataTracker) nextSeq(dev protocol.DeviceID) int64 { // devices returns the list of devices tracked, excluding the local device // (which we don't know the ID of) func (m *metadataTracker) devices() []protocol.DeviceID { - devs := make([]protocol.DeviceID, 0, len(m.counts.Counts)) + devs := make(map[protocol.DeviceID]struct{}, len(m.counts.Counts)) m.mut.RLock() for _, dev := range m.counts.Counts { if dev.Sequence > 0 { id := protocol.DeviceIDFromBytes(dev.DeviceID) - if id == globalDeviceID || id == protocol.LocalDeviceID { + if id == protocol.GlobalDeviceID || id == protocol.LocalDeviceID { continue } - devs = append(devs, id) + devs[id] = struct{}{} } } m.mut.RUnlock() - return devs + devList := make([]protocol.DeviceID, 0, len(devs)) + for dev := range devs { + devList = append(devList, dev) + } + + return devList } func (m *metadataTracker) Created() time.Time { @@ -234,3 +277,19 @@ func (m *metadataTracker) SetCreated() { m.counts.Created = time.Now().UnixNano() m.mut.Unlock() } + +// eachFlagBit calls the function once for every bit that is set in flags +func eachFlagBit(flags uint32, fn func(flag uint32)) { + // Test each bit from the right, as long as there are bits left in the + // flag set. Clear any bits found and stop testing as soon as there are + // no more bits set. + + currentBit := uint32(1 << 0) + for flags != 0 { + if flags¤tBit != 0 { + fn(currentBit) + flags &^= currentBit + } + currentBit <<= 1 + } +} diff --git a/lib/db/meta_test.go b/lib/db/meta_test.go new file mode 100644 index 000000000..4d807f656 --- /dev/null +++ b/lib/db/meta_test.go @@ -0,0 +1,82 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package db + +import ( + "math/bits" + "sort" + "testing" + + "github.com/syncthing/syncthing/lib/protocol" +) + +func TestEachFlagBit(t *testing.T) { + cases := []struct { + flags uint32 + iterations int + }{ + {0, 0}, + {1<<0 | 1<<3, 2}, + {1 << 0, 1}, + {1 << 31, 1}, + {1<<10 | 1<<20 | 1<<30, 3}, + } + + for _, tc := range cases { + var flags uint32 + iterations := 0 + + eachFlagBit(tc.flags, func(f uint32) { + iterations++ + flags |= f + if bits.OnesCount32(f) != 1 { + t.Error("expected exactly one bit to be set in every call") + } + }) + + if flags != tc.flags { + t.Errorf("expected 0x%x flags, got 0x%x", tc.flags, flags) + } + if iterations != tc.iterations { + t.Errorf("expected %d iterations, got %d", tc.iterations, iterations) + } + } +} + +func TestMetaDevices(t *testing.T) { + d1 := protocol.DeviceID{1} + d2 := protocol.DeviceID{2} + meta := newMetadataTracker() + + meta.addFile(d1, protocol.FileInfo{Sequence: 1}) + meta.addFile(d1, protocol.FileInfo{Sequence: 2, LocalFlags: 1}) + meta.addFile(d2, protocol.FileInfo{Sequence: 1}) + meta.addFile(d2, protocol.FileInfo{Sequence: 2, LocalFlags: 2}) + meta.addFile(protocol.LocalDeviceID, protocol.FileInfo{Sequence: 1}) + + // There are five device/flags combos + if l := len(meta.counts.Counts); l < 5 { + t.Error("expected at least five buckets, not", l) + } + + // There are only two non-local devices + devs := meta.devices() + if l := len(devs); l != 2 { + t.Fatal("expected two devices, not", l) + } + + // Check that we got the two devices we expect + sort.Slice(devs, func(a, b int) bool { + return devs[a].Compare(devs[b]) == -1 + }) + if devs[0] != d1 { + t.Error("first device should be d1") + } + if devs[1] != d2 { + t.Error("second device should be d2") + } +} diff --git a/lib/db/set.go b/lib/db/set.go index 314ed4538..8093fb3c0 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -37,8 +37,12 @@ type FileSet struct { type FileIntf interface { FileSize() int64 FileName() string + FileLocalFlags() uint32 IsDeleted() bool IsInvalid() bool + IsIgnored() bool + IsUnsupported() bool + MustRescan() bool IsDirectory() bool IsSymlink() bool HasPermissionBits() bool @@ -248,15 +252,23 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID { } func (s *FileSet) Sequence(device protocol.DeviceID) int64 { - return s.meta.Counts(device).Sequence + return s.meta.Counts(device, 0).Sequence } func (s *FileSet) LocalSize() Counts { - return s.meta.Counts(protocol.LocalDeviceID) + local := s.meta.Counts(protocol.LocalDeviceID, 0) + recvOnlyChanged := s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) + return local.Add(recvOnlyChanged) +} + +func (s *FileSet) ReceiveOnlyChangedSize() Counts { + return s.meta.Counts(protocol.LocalDeviceID, protocol.FlagLocalReceiveOnly) } func (s *FileSet) GlobalSize() Counts { - return s.meta.Counts(globalDeviceID) + global := s.meta.Counts(protocol.GlobalDeviceID, 0) + recvOnlyChanged := s.meta.Counts(protocol.GlobalDeviceID, protocol.FlagLocalReceiveOnly) + return global.Add(recvOnlyChanged) } func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID { diff --git a/lib/db/set_test.go b/lib/db/set_test.go index fd0cb540e..53087a579 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -906,7 +906,7 @@ func TestWithHaveSequence(t *testing.T) { i := 2 s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { - if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], false, false) { + if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) { t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) } i++ @@ -917,7 +917,7 @@ func TestWithHaveSequence(t *testing.T) { func TestIssue4925(t *testing.T) { ldb := db.OpenMemory() - folder := "test)" + folder := "test" s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) localHave := fileList{ @@ -955,7 +955,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Error("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(remote0Have[0], false, false) { + } else if !need[0].IsEquivalent(remote0Have[0]) { t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0]) } @@ -981,7 +981,7 @@ func TestMoveGlobalBack(t *testing.T) { if need := needList(s, remoteDevice0); len(need) != 1 { t.Error("Expected 1 need for remote 0, got", need) - } else if !need[0].IsEquivalent(localHave[0], false, false) { + } else if !need[0].IsEquivalent(localHave[0]) { t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0]) } @@ -1017,7 +1017,7 @@ func TestIssue5007(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0], false, false) { + } else if !need[0].IsEquivalent(fs[0]) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1052,7 +1052,7 @@ func TestNeedDeleted(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected 1 local need, got", need) - } else if !need[0].IsEquivalent(fs[0], false, false) { + } else if !need[0].IsEquivalent(fs[0]) { t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0]) } @@ -1065,6 +1065,110 @@ func TestNeedDeleted(t *testing.T) { } } +func TestReceiveOnlyAccounting(t *testing.T) { + ldb := db.OpenMemory() + + folder := "test" + s := db.NewFileSet(folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb) + + local := protocol.DeviceID{1} + remote := protocol.DeviceID{2} + + // Three files that have been created by the remote device + + version := protocol.Vector{Counters: []protocol.Counter{{ID: remote.Short(), Value: 1}}} + files := fileList{ + protocol.FileInfo{Name: "f1", Size: 10, Sequence: 1, Version: version}, + protocol.FileInfo{Name: "f2", Size: 10, Sequence: 1, Version: version}, + protocol.FileInfo{Name: "f3", Size: 10, Sequence: 1, Version: version}, + } + + // We have synced them locally + + replace(s, protocol.LocalDeviceID, files) + replace(s, remote, files) + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files initially, not", n) + } + if n := s.LocalSize().Bytes; n != 30 { + t.Fatal("expected 30 local bytes initially, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files initially, not", n) + } + if n := s.GlobalSize().Bytes; n != 30 { + t.Fatal("expected 30 global bytes initially, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + t.Fatal("expected 0 receive only changed files initially, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + t.Fatal("expected 0 receive only changed bytes initially, not", n) + } + + // Detected a local change in a receive only folder + + changed := files[0] + changed.Version = changed.Version.Update(local.Short()) + changed.Size = 100 + changed.ModifiedBy = local.Short() + changed.LocalFlags = protocol.FlagLocalReceiveOnly + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed}) + + // Check that we see the files + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files after local change, not", n) + } + if n := s.LocalSize().Bytes; n != 120 { + t.Fatal("expected 120 local bytes after local change, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files after local change, not", n) + } + if n := s.GlobalSize().Bytes; n != 120 { + t.Fatal("expected 120 global bytes after local change, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 1 { + t.Fatal("expected 1 receive only changed file after local change, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 100 { + t.Fatal("expected 100 receive only changed btyes after local change, not", n) + } + + // Fake a revert. That's a two step process, first converting our + // changed file into a less preferred variant, then pulling down the old + // version. + + changed.Version = protocol.Vector{} + changed.LocalFlags &^= protocol.FlagLocalReceiveOnly + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed}) + + s.Update(protocol.LocalDeviceID, []protocol.FileInfo{files[0]}) + + // Check that we see the files, same data as initially + + if n := s.LocalSize().Files; n != 3 { + t.Fatal("expected 3 local files after revert, not", n) + } + if n := s.LocalSize().Bytes; n != 30 { + t.Fatal("expected 30 local bytes after revert, not", n) + } + if n := s.GlobalSize().Files; n != 3 { + t.Fatal("expected 3 global files after revert, not", n) + } + if n := s.GlobalSize().Bytes; n != 30 { + t.Fatal("expected 30 global bytes after revert, not", n) + } + if n := s.ReceiveOnlyChangedSize().Files; n != 0 { + t.Fatal("expected 0 receive only changed files after revert, not", n) + } + if n := s.ReceiveOnlyChangedSize().Bytes; n != 0 { + t.Fatal("expected 0 receive only changed bytes after revert, not", n) + } +} + func TestNeedAfterUnignore(t *testing.T) { ldb := db.OpenMemory() @@ -1090,7 +1194,7 @@ func TestNeedAfterUnignore(t *testing.T) { if need := needList(s, protocol.LocalDeviceID); len(need) != 1 { t.Fatal("Expected one local need, got", need) - } else if !need[0].IsEquivalent(remote, false, false) { + } else if !need[0].IsEquivalent(remote) { t.Fatalf("Got %v, expected %v", need[0], remote) } } diff --git a/lib/db/structs.go b/lib/db/structs.go index 91aff8409..985a432dd 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -40,6 +40,10 @@ func (f FileInfoTruncated) IsInvalid() bool { return f.RawInvalid || f.LocalFlags&protocol.LocalInvalidFlags != 0 } +func (f FileInfoTruncated) IsUnsupported() bool { + return f.LocalFlags&protocol.FlagLocalUnsupported != 0 +} + func (f FileInfoTruncated) IsIgnored() bool { return f.LocalFlags&protocol.FlagLocalIgnored != 0 } @@ -48,6 +52,10 @@ func (f FileInfoTruncated) MustRescan() bool { return f.LocalFlags&protocol.FlagLocalMustRescan != 0 } +func (f FileInfoTruncated) IsReceiveOnlyChanged() bool { + return f.LocalFlags&protocol.FlagLocalReceiveOnly != 0 +} + func (f FileInfoTruncated) IsDirectory() bool { return f.Type == protocol.FileInfoTypeDirectory } @@ -86,6 +94,10 @@ func (f FileInfoTruncated) FileName() string { return f.Name } +func (f FileInfoTruncated) FileLocalFlags() uint32 { + return f.LocalFlags +} + func (f FileInfoTruncated) ModTime() time.Time { return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) } @@ -110,3 +122,16 @@ func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protoco LocalFlags: protocol.FlagLocalIgnored, } } + +func (c Counts) Add(other Counts) Counts { + return Counts{ + Files: c.Files + other.Files, + Directories: c.Directories + other.Directories, + Symlinks: c.Symlinks + other.Symlinks, + Deleted: c.Deleted + other.Deleted, + Bytes: c.Bytes + other.Bytes, + Sequence: c.Sequence + other.Sequence, + DeviceID: protocol.EmptyDeviceID[:], + LocalFlags: c.LocalFlags | other.LocalFlags, + } +} diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index e11fc5359..e3a9e4b7e 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -91,6 +91,7 @@ type Counts struct { Bytes int64 `protobuf:"varint,5,opt,name=bytes,proto3" json:"bytes,omitempty"` Sequence int64 `protobuf:"varint,6,opt,name=sequence,proto3" json:"sequence,omitempty"` DeviceID []byte `protobuf:"bytes,17,opt,name=deviceID,proto3" json:"deviceID,omitempty"` + LocalFlags uint32 `protobuf:"varint,18,opt,name=localFlags,proto3" json:"localFlags,omitempty"` } func (m *Counts) Reset() { *m = Counts{} } @@ -357,6 +358,13 @@ func (m *Counts) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintStructs(dAtA, i, uint64(len(m.DeviceID))) i += copy(dAtA[i:], m.DeviceID) } + if m.LocalFlags != 0 { + dAtA[i] = 0x90 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintStructs(dAtA, i, uint64(m.LocalFlags)) + } return i, nil } @@ -526,6 +534,9 @@ func (m *Counts) ProtoSize() (n int) { if l > 0 { n += 2 + l + sovStructs(uint64(l)) } + if m.LocalFlags != 0 { + n += 2 + sovStructs(uint64(m.LocalFlags)) + } return n } @@ -1312,6 +1323,25 @@ func (m *Counts) Unmarshal(dAtA []byte) error { m.DeviceID = []byte{} } iNdEx = postIndex + case 18: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LocalFlags", wireType) + } + m.LocalFlags = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LocalFlags |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStructs(dAtA[iNdEx:]) @@ -1541,47 +1571,47 @@ var ( func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) } var fileDescriptorStructs = []byte{ - // 663 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x40, - 0x10, 0xb6, 0x62, 0xf9, 0x6f, 0x6c, 0xa7, 0xc9, 0x12, 0x82, 0x30, 0xd4, 0x16, 0x86, 0x82, 0x28, - 0xd4, 0x6e, 0x13, 0x7a, 0x69, 0x6f, 0x6a, 0x08, 0x18, 0x4a, 0x5b, 0xd6, 0x21, 0xa7, 0x82, 0xd1, - 0xcf, 0xda, 0x59, 0x22, 0x6b, 0x1d, 0xed, 0x3a, 0x41, 0x79, 0x92, 0x1e, 0xf3, 0x30, 0x3d, 0xe4, - 0xd8, 0x73, 0x0f, 0x26, 0x75, 0x2f, 0x7d, 0x8c, 0xb2, 0xbb, 0x92, 0xa2, 0xf6, 0xd4, 0xde, 0xe6, - 0x9b, 0x9f, 0x9d, 0x6f, 0x66, 0xbe, 0x85, 0x2e, 0x17, 0xc9, 0x3a, 0x10, 0x7c, 0xb4, 0x4a, 0x98, - 0x60, 0x68, 0x27, 0xf4, 0x7b, 0x2f, 0x16, 0x54, 0x5c, 0xac, 0xfd, 0x51, 0xc0, 0x96, 0xe3, 0x05, - 0x5b, 0xb0, 0xb1, 0x0a, 0xf9, 0xeb, 0xb9, 0x42, 0x0a, 0x28, 0x4b, 0x97, 0xf4, 0x5e, 0x97, 0xd2, - 0x79, 0x1a, 0x07, 0xe2, 0x82, 0xc6, 0x8b, 0x92, 0x15, 0x51, 0x5f, 0xbf, 0x10, 0xb0, 0x68, 0xec, - 0x93, 0x95, 0x2e, 0x1b, 0x5e, 0x41, 0xfb, 0x94, 0x46, 0xe4, 0x9c, 0x24, 0x9c, 0xb2, 0x18, 0xbd, - 0x84, 0xc6, 0xb5, 0x36, 0x2d, 0xc3, 0x36, 0x9c, 0xf6, 0xd1, 0xde, 0x28, 0x2f, 0x1a, 0x9d, 0x93, - 0x40, 0xb0, 0xc4, 0x35, 0xef, 0x37, 0x83, 0x0a, 0xce, 0xd3, 0xd0, 0x21, 0xd4, 0x43, 0x72, 0x4d, - 0x03, 0x62, 0xed, 0xd8, 0x86, 0xd3, 0xc1, 0x19, 0x42, 0x16, 0x34, 0x68, 0x7c, 0xed, 0x45, 0x34, - 0xb4, 0xaa, 0xb6, 0xe1, 0x34, 0x71, 0x0e, 0x87, 0xa7, 0xd0, 0xce, 0xda, 0xbd, 0xa7, 0x5c, 0xa0, - 0x57, 0xd0, 0xcc, 0xde, 0xe2, 0x96, 0x61, 0x57, 0x9d, 0xf6, 0xd1, 0x93, 0x51, 0xe8, 0x8f, 0x4a, - 0xac, 0xb2, 0x96, 0x45, 0xda, 0x1b, 0xf3, 0xcb, 0xdd, 0xa0, 0x32, 0x7c, 0x30, 0x61, 0x5f, 0x66, - 0x4d, 0xe2, 0x39, 0x3b, 0x4b, 0xd6, 0x71, 0xe0, 0x09, 0x12, 0x22, 0x04, 0x66, 0xec, 0x2d, 0x89, - 0xa2, 0xdf, 0xc2, 0xca, 0x46, 0xcf, 0xc1, 0x14, 0xe9, 0x4a, 0x33, 0xdc, 0x3d, 0x3a, 0x7c, 0x1c, - 0xa9, 0x28, 0x4f, 0x57, 0x04, 0xab, 0x1c, 0x59, 0xcf, 0xe9, 0x2d, 0x51, 0xa4, 0xab, 0x58, 0xd9, - 0xc8, 0x86, 0xf6, 0x8a, 0x24, 0x4b, 0xca, 0x35, 0x4b, 0xd3, 0x36, 0x9c, 0x2e, 0x2e, 0xbb, 0xd0, - 0x53, 0x80, 0x25, 0x0b, 0xe9, 0x9c, 0x92, 0x70, 0xc6, 0xad, 0x9a, 0xaa, 0x6d, 0xe5, 0x9e, 0xa9, - 0x5c, 0x46, 0x48, 0x22, 0x22, 0x48, 0x68, 0xd5, 0xf5, 0x32, 0x32, 0x88, 0x9c, 0xc7, 0x35, 0x35, - 0x64, 0xc4, 0xdd, 0xdd, 0x6e, 0x06, 0x80, 0xbd, 0x9b, 0x89, 0xf6, 0x16, 0x6b, 0x43, 0xcf, 0x60, - 0x37, 0x66, 0xb3, 0x32, 0x8f, 0xa6, 0x7a, 0xaa, 0x1b, 0xb3, 0x4f, 0x25, 0x26, 0xa5, 0x0b, 0xb6, - 0xfe, 0xed, 0x82, 0x3d, 0x68, 0x72, 0x72, 0xb5, 0x26, 0x71, 0x40, 0x2c, 0x50, 0xcc, 0x0b, 0x8c, - 0x06, 0xd0, 0x2e, 0xe6, 0x8a, 0xb9, 0xd5, 0xb6, 0x0d, 0xa7, 0x86, 0x8b, 0x51, 0x3f, 0x70, 0xf4, - 0xb9, 0x94, 0xe0, 0xa7, 0x56, 0xc7, 0x36, 0x1c, 0xd3, 0x7d, 0x2b, 0x1b, 0x7c, 0xdf, 0x0c, 0x8e, - 0xff, 0x43, 0x93, 0xa3, 0xe9, 0x05, 0x4b, 0xc4, 0xe4, 0xe4, 0xf1, 0x75, 0x37, 0x45, 0x63, 0x00, - 0x3f, 0x62, 0xc1, 0xe5, 0x4c, 0x9d, 0xa4, 0x2b, 0xbb, 0xbb, 0x7b, 0xdb, 0xcd, 0xa0, 0x83, 0xbd, - 0x1b, 0x57, 0x06, 0xa6, 0xf4, 0x96, 0xe0, 0x96, 0x9f, 0x9b, 0x72, 0x49, 0x3c, 0x5d, 0x46, 0x34, - 0xbe, 0x9c, 0x09, 0x2f, 0x59, 0x10, 0x61, 0xed, 0x2b, 0x1d, 0x74, 0x33, 0xef, 0x99, 0x72, 0xca, - 0x83, 0x46, 0x2c, 0xf0, 0xa2, 0xd9, 0x3c, 0xf2, 0x16, 0xdc, 0xfa, 0xd5, 0x50, 0x17, 0x05, 0xe5, - 0x3b, 0x95, 0xae, 0x4c, 0x62, 0x5f, 0x0d, 0xa8, 0xbf, 0x63, 0xeb, 0x58, 0x70, 0x74, 0x00, 0xb5, - 0x39, 0x8d, 0x08, 0x57, 0xc2, 0xaa, 0x61, 0x0d, 0xe4, 0x43, 0x21, 0x4d, 0xd4, 0x5a, 0x29, 0xe1, - 0x4a, 0x60, 0x35, 0x5c, 0x76, 0xa9, 0xed, 0xea, 0xde, 0x5c, 0x69, 0xaa, 0x86, 0x0b, 0x5c, 0x96, - 0x85, 0xa9, 0x42, 0x85, 0x2c, 0x0e, 0xa0, 0xe6, 0xa7, 0x82, 0xe4, 0x52, 0xd2, 0xe0, 0x8f, 0x4b, - 0xd5, 0xff, 0xba, 0x54, 0x0f, 0x9a, 0xfa, 0xe7, 0x4d, 0x4e, 0xd4, 0xcc, 0x1d, 0x5c, 0xe0, 0xe1, - 0x47, 0x68, 0xe9, 0x29, 0xa6, 0x44, 0x20, 0x07, 0xea, 0x81, 0x02, 0xd9, 0x6f, 0x03, 0xf9, 0xdb, - 0x74, 0x38, 0x53, 0x46, 0x16, 0x97, 0xf4, 0x82, 0x84, 0xc8, 0x5f, 0xa5, 0x06, 0xab, 0xe2, 0x1c, - 0xba, 0x07, 0xf7, 0x3f, 0xfa, 0x95, 0xfb, 0x6d, 0xdf, 0xf8, 0xb6, 0xed, 0x1b, 0x0f, 0xdb, 0x7e, - 0xe5, 0xee, 0x67, 0xdf, 0xf0, 0xeb, 0xea, 0x96, 0xc7, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x9a, - 0x4b, 0x16, 0x44, 0xcd, 0x04, 0x00, 0x00, + // 671 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6b, 0xdb, 0x4c, + 0x10, 0xb6, 0x62, 0xf9, 0x6b, 0x6c, 0xe7, 0x4d, 0x96, 0x10, 0x84, 0xe1, 0xb5, 0x85, 0xa1, 0x20, + 0x0a, 0xb5, 0xdb, 0x84, 0x5e, 0xda, 0x9b, 0x1a, 0x02, 0x86, 0xd2, 0x96, 0x75, 0xc8, 0xa9, 0x60, + 0xf4, 0xb1, 0x76, 0x96, 0xc8, 0x5a, 0x47, 0xbb, 0x4e, 0x50, 0x7e, 0x49, 0x8f, 0xf9, 0x39, 0x39, + 0xf6, 0xdc, 0x83, 0x49, 0xdd, 0x1e, 0xfa, 0x33, 0xca, 0xee, 0x4a, 0x8a, 0x9a, 0x53, 0x7b, 0x9b, + 0x67, 0x3e, 0x76, 0x9e, 0x99, 0x79, 0x16, 0xba, 0x5c, 0x24, 0xeb, 0x40, 0xf0, 0xd1, 0x2a, 0x61, + 0x82, 0xa1, 0x9d, 0xd0, 0xef, 0xbd, 0x58, 0x50, 0x71, 0xb1, 0xf6, 0x47, 0x01, 0x5b, 0x8e, 0x17, + 0x6c, 0xc1, 0xc6, 0x2a, 0xe4, 0xaf, 0xe7, 0x0a, 0x29, 0xa0, 0x2c, 0x5d, 0xd2, 0x7b, 0x5d, 0x4a, + 0xe7, 0x69, 0x1c, 0x88, 0x0b, 0x1a, 0x2f, 0x4a, 0x56, 0x44, 0x7d, 0xfd, 0x42, 0xc0, 0xa2, 0xb1, + 0x4f, 0x56, 0xba, 0x6c, 0x78, 0x05, 0xed, 0x53, 0x1a, 0x91, 0x73, 0x92, 0x70, 0xca, 0x62, 0xf4, + 0x12, 0x1a, 0xd7, 0xda, 0xb4, 0x0c, 0xdb, 0x70, 0xda, 0x47, 0x7b, 0xa3, 0xbc, 0x68, 0x74, 0x4e, + 0x02, 0xc1, 0x12, 0xd7, 0xbc, 0xdf, 0x0c, 0x2a, 0x38, 0x4f, 0x43, 0x87, 0x50, 0x0f, 0xc9, 0x35, + 0x0d, 0x88, 0xb5, 0x63, 0x1b, 0x4e, 0x07, 0x67, 0x08, 0x59, 0xd0, 0xa0, 0xf1, 0xb5, 0x17, 0xd1, + 0xd0, 0xaa, 0xda, 0x86, 0xd3, 0xc4, 0x39, 0x1c, 0x9e, 0x42, 0x3b, 0x6b, 0xf7, 0x9e, 0x72, 0x81, + 0x5e, 0x41, 0x33, 0x7b, 0x8b, 0x5b, 0x86, 0x5d, 0x75, 0xda, 0x47, 0xff, 0x8d, 0x42, 0x7f, 0x54, + 0x62, 0x95, 0xb5, 0x2c, 0xd2, 0xde, 0x98, 0x5f, 0xee, 0x06, 0x95, 0xe1, 0x83, 0x09, 0xfb, 0x32, + 0x6b, 0x12, 0xcf, 0xd9, 0x59, 0xb2, 0x8e, 0x03, 0x4f, 0x90, 0x10, 0x21, 0x30, 0x63, 0x6f, 0x49, + 0x14, 0xfd, 0x16, 0x56, 0x36, 0x7a, 0x0e, 0xa6, 0x48, 0x57, 0x9a, 0xe1, 0xee, 0xd1, 0xe1, 0xe3, + 0x48, 0x45, 0x79, 0xba, 0x22, 0x58, 0xe5, 0xc8, 0x7a, 0x4e, 0x6f, 0x89, 0x22, 0x5d, 0xc5, 0xca, + 0x46, 0x36, 0xb4, 0x57, 0x24, 0x59, 0x52, 0xae, 0x59, 0x9a, 0xb6, 0xe1, 0x74, 0x71, 0xd9, 0x85, + 0xfe, 0x07, 0x58, 0xb2, 0x90, 0xce, 0x29, 0x09, 0x67, 0xdc, 0xaa, 0xa9, 0xda, 0x56, 0xee, 0x99, + 0xca, 0x65, 0x84, 0x24, 0x22, 0x82, 0x84, 0x56, 0x5d, 0x2f, 0x23, 0x83, 0xc8, 0x79, 0x5c, 0x53, + 0x43, 0x46, 0xdc, 0xdd, 0xed, 0x66, 0x00, 0xd8, 0xbb, 0x99, 0x68, 0x6f, 0xb1, 0x36, 0xf4, 0x0c, + 0x76, 0x63, 0x36, 0x2b, 0xf3, 0x68, 0xaa, 0xa7, 0xba, 0x31, 0xfb, 0x54, 0x62, 0x52, 0xba, 0x60, + 0xeb, 0xef, 0x2e, 0xd8, 0x83, 0x26, 0x27, 0x57, 0x6b, 0x12, 0x07, 0xc4, 0x02, 0xc5, 0xbc, 0xc0, + 0x68, 0x00, 0xed, 0x62, 0xae, 0x98, 0x5b, 0x6d, 0xdb, 0x70, 0x6a, 0xb8, 0x18, 0xf5, 0x03, 0x47, + 0x9f, 0x4b, 0x09, 0x7e, 0x6a, 0x75, 0x6c, 0xc3, 0x31, 0xdd, 0xb7, 0xb2, 0xc1, 0xb7, 0xcd, 0xe0, + 0xf8, 0x1f, 0x34, 0x39, 0x9a, 0x5e, 0xb0, 0x44, 0x4c, 0x4e, 0x1e, 0x5f, 0x77, 0x53, 0x34, 0x06, + 0xf0, 0x23, 0x16, 0x5c, 0xce, 0xd4, 0x49, 0xba, 0xb2, 0xbb, 0xbb, 0xb7, 0xdd, 0x0c, 0x3a, 0xd8, + 0xbb, 0x71, 0x65, 0x60, 0x4a, 0x6f, 0x09, 0x6e, 0xf9, 0xb9, 0x29, 0x97, 0xc4, 0xd3, 0x65, 0x44, + 0xe3, 0xcb, 0x99, 0xf0, 0x92, 0x05, 0x11, 0xd6, 0xbe, 0xd2, 0x41, 0x37, 0xf3, 0x9e, 0x29, 0xa7, + 0x3c, 0x68, 0xc4, 0x02, 0x2f, 0x9a, 0xcd, 0x23, 0x6f, 0xc1, 0xad, 0x5f, 0x0d, 0x75, 0x51, 0x50, + 0xbe, 0x53, 0xe9, 0xca, 0x24, 0xf6, 0xd3, 0x80, 0xfa, 0x3b, 0xb6, 0x8e, 0x05, 0x47, 0x07, 0x50, + 0x9b, 0xd3, 0x88, 0x70, 0x25, 0xac, 0x1a, 0xd6, 0x40, 0x3e, 0x14, 0xd2, 0x44, 0xad, 0x95, 0x12, + 0xae, 0x04, 0x56, 0xc3, 0x65, 0x97, 0xda, 0xae, 0xee, 0xcd, 0x95, 0xa6, 0x6a, 0xb8, 0xc0, 0x65, + 0x59, 0x98, 0x2a, 0x54, 0xc8, 0xe2, 0x00, 0x6a, 0x7e, 0x2a, 0x48, 0x2e, 0x25, 0x0d, 0xfe, 0xb8, + 0x54, 0xfd, 0xc9, 0xa5, 0x7a, 0xd0, 0xd4, 0x3f, 0x6f, 0x72, 0xa2, 0x66, 0xee, 0xe0, 0x02, 0xa3, + 0x3e, 0x94, 0x46, 0xb3, 0xd0, 0xd3, 0x61, 0x87, 0x1f, 0xa1, 0xa5, 0xa7, 0x9c, 0x12, 0x81, 0x1c, + 0xa8, 0x07, 0x0a, 0x64, 0xbf, 0x11, 0xe4, 0x6f, 0xd4, 0xe1, 0x4c, 0x39, 0x59, 0x5c, 0xd2, 0x0f, + 0x12, 0x22, 0x7f, 0x9d, 0x1a, 0xbc, 0x8a, 0x73, 0xe8, 0x1e, 0xdc, 0x7f, 0xef, 0x57, 0xee, 0xb7, + 0x7d, 0xe3, 0xeb, 0xb6, 0x6f, 0x3c, 0x6c, 0xfb, 0x95, 0xbb, 0x1f, 0x7d, 0xc3, 0xaf, 0xab, 0x5b, + 0x1f, 0xff, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x4d, 0xd7, 0x14, 0xed, 0x04, 0x00, 0x00, } diff --git a/lib/db/structs.proto b/lib/db/structs.proto index 03309f1cc..9794adb32 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -46,13 +46,14 @@ message FileInfoTruncated { // For each folder and device we keep one of these to track the current // counts and sequence. We also keep one for the global state of the folder. message Counts { - int32 files = 1; - int32 directories = 2; - int32 symlinks = 3; - int32 deleted = 4; - int64 bytes = 5; - int64 sequence = 6; // zero for the global state - bytes deviceID = 17; // device ID for remote devices, or special values for local/global + int32 files = 1; + int32 directories = 2; + int32 symlinks = 3; + int32 deleted = 4; + int64 bytes = 5; + int64 sequence = 6; // zero for the global state + bytes deviceID = 17; // device ID for remote devices, or special values for local/global + uint32 localFlags = 18; // the local flag for this count bucket } message CountsSet { diff --git a/lib/model/folder.go b/lib/model/folder.go index fd425f598..9716e272e 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -27,6 +27,7 @@ var errWatchNotStarted = errors.New("not started") type folder struct { stateTracker config.FolderConfiguration + localFlags uint32 model *Model shortID protocol.ShortID @@ -175,6 +176,8 @@ func (f *folder) BringToFront(string) {} func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} +func (f *folder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {} + func (f *folder) DelayScan(next time.Duration) { f.Delay(next) } @@ -263,7 +266,7 @@ func (f *folder) getHealthError() error { } func (f *folder) scanSubdirs(subDirs []string) error { - if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil { + if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs, f.localFlags); err != nil { // Potentially sets the error twice, once in the scanner just // by doing a check, and once here, if the error returned is // the same one as returned by CheckHealth, though diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go new file mode 100644 index 000000000..7638b9437 --- /dev/null +++ b/lib/model/folder_recvonly.go @@ -0,0 +1,210 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package model + +import ( + "sort" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/ignore" + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/versioner" +) + +func init() { + folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder +} + +/* +receiveOnlyFolder is a folder that does not propagate local changes outward. +It does this by the following general mechanism (not all of which is +implemted in this file): + +- Local changes are scanned and versioned as usual, but get the + FlagLocalReceiveOnly bit set. + +- When changes are sent to the cluster this bit gets converted to the + Invalid bit (like all other local flags, currently) and also the Version + gets set to the empty version. The reason for clearing the Version is to + ensure that other devices will not consider themselves out of date due to + our change. + +- The database layer accounts sizes per flag bit, so we can know how many + files have been changed locally. We use this to trigger a "Revert" option + on the folder when the amount of locally changed data is nonzero. + +- To revert we take the files which have changed and reset their version + counter down to zero. The next pull will replace our changed version with + the globally latest. As this is a user-initiated operation we do not cause + conflict copies when reverting. + +- When pulling normally (i.e., not in the revert case) with local changes, + normal conflict resolution will apply. Conflict copies will be created, + but not propagated outwards (because receive only, right). + +Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that +sets an extra bit on local changes and has a Revert method. +*/ +type receiveOnlyFolder struct { + *sendReceiveFolder +} + +func newReceiveOnlyFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { + sr := newSendReceiveFolder(model, cfg, ver, fs).(*sendReceiveFolder) + sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files + return &receiveOnlyFolder{sr} +} + +func (f *receiveOnlyFolder) Revert(fs *db.FileSet, updateFn func([]protocol.FileInfo)) { + f.setState(FolderScanning) + defer f.setState(FolderIdle) + + // XXX: This *really* should be given to us in the constructor... + f.model.fmut.RLock() + ignores := f.model.folderIgnores[f.folderID] + f.model.fmut.RUnlock() + + delQueue := &deleteQueue{ + handler: f, // for the deleteFile and deleteDir methods + ignores: ignores, + } + + batch := make([]protocol.FileInfo, 0, maxBatchSizeFiles) + batchSizeBytes := 0 + fs.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + fi := intf.(protocol.FileInfo) + if !fi.IsReceiveOnlyChanged() { + // We're only interested in files that have changed locally in + // receive only mode. + return true + } + + if len(fi.Version.Counters) == 1 && fi.Version.Counters[0].ID == f.shortID { + // We are the only device mentioned in the version vector so the + // file must originate here. A revert then means to delete it. + // We'll delete files directly, directories get queued and + // handled below. + + handled, err := delQueue.handle(fi) + if err != nil { + l.Infof("Revert: deleting %s: %v\n", fi.Name, err) + return true // continue + } + if !handled { + return true // continue + } + + fi = protocol.FileInfo{ + Name: fi.Name, + Type: fi.Type, + ModifiedS: fi.ModifiedS, + ModifiedNs: fi.ModifiedNs, + ModifiedBy: f.shortID, + Deleted: true, + Version: protocol.Vector{}, // if this file ever resurfaces anywhere we want our delete to be strictly older + } + } else { + // Revert means to throw away our local changes. We reset the + // version to the empty vector, which is strictly older than any + // other existing version. It is not in conflict with anything, + // either, so we will not create a conflict copy of our local + // changes. + fi.Version = protocol.Vector{} + fi.LocalFlags &^= protocol.FlagLocalReceiveOnly + } + + batch = append(batch, fi) + batchSizeBytes += fi.ProtoSize() + + if len(batch) >= maxBatchSizeFiles || batchSizeBytes >= maxBatchSizeBytes { + updateFn(batch) + batch = batch[:0] + batchSizeBytes = 0 + } + return true + }) + if len(batch) > 0 { + updateFn(batch) + } + batch = batch[:0] + batchSizeBytes = 0 + + // Handle any queued directories + deleted, err := delQueue.flush() + if err != nil { + l.Infoln("Revert:", err) + } + now := time.Now() + for _, dir := range deleted { + batch = append(batch, protocol.FileInfo{ + Name: dir, + Type: protocol.FileInfoTypeDirectory, + ModifiedS: now.Unix(), + ModifiedBy: f.shortID, + Deleted: true, + Version: protocol.Vector{}, + }) + } + if len(batch) > 0 { + updateFn(batch) + } + + // We will likely have changed our local index, but that won't trigger a + // pull by itself. Make sure we schedule one so that we start + // downloading files. + f.SchedulePull() +} + +// deleteQueue handles deletes by delegating to a handler and queuing +// directories for last. +type deleteQueue struct { + handler interface { + deleteFile(file protocol.FileInfo) (dbUpdateJob, error) + deleteDir(dir string, ignores *ignore.Matcher, scanChan chan<- string) error + } + ignores *ignore.Matcher + dirs []string +} + +func (q *deleteQueue) handle(fi protocol.FileInfo) (bool, error) { + // Things that are ignored but not marked deletable are not processed. + ign := q.ignores.Match(fi.Name) + if ign.IsIgnored() && !ign.IsDeletable() { + return false, nil + } + + // Directories are queued for later processing. + if fi.IsDirectory() { + q.dirs = append(q.dirs, fi.Name) + return false, nil + } + + // Kill it. + _, err := q.handler.deleteFile(fi) + return true, err +} + +func (q *deleteQueue) flush() ([]string, error) { + // Process directories from the leaves inward. + sort.Sort(sort.Reverse(sort.StringSlice(q.dirs))) + + var firstError error + var deleted []string + + for _, dir := range q.dirs { + if err := q.handler.deleteDir(dir, q.ignores, nil); err == nil { + deleted = append(deleted, dir) + } else if err != nil && firstError == nil { + firstError = err + } + } + + return deleted, firstError +} diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go new file mode 100644 index 000000000..99591d410 --- /dev/null +++ b/lib/model/folder_recvonly_test.go @@ -0,0 +1,261 @@ +// Copyright (C) 2018 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package model + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/protocol" +) + +func TestRecvOnlyRevertDeletes(t *testing.T) { + // Make sure that we delete extraneous files and directories when we hit + // Revert. + + os.RemoveAll("_recvonly") + defer os.RemoveAll("_recvonly") + + // Create some test data + + os.MkdirAll("_recvonly/.stfolder", 0755) + os.MkdirAll("_recvonly/ignDir", 0755) + os.MkdirAll("_recvonly/unknownDir", 0755) + ioutil.WriteFile("_recvonly/ignDir/ignFile", []byte("hello\n"), 0644) + ioutil.WriteFile("_recvonly/unknownDir/unknownFile", []byte("hello\n"), 0644) + ioutil.WriteFile("_recvonly/.stignore", []byte("ignDir\n"), 0644) + + knownFiles := setupKnownFiles(t, []byte("hello\n")) + + // Get us a model up and running + + m := setupROFolder() + defer m.Stop() + + // Send and index update for the known stuff + + m.Index(device1, "ro", knownFiles) + m.updateLocalsFromScanning("ro", knownFiles) + + size := m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) + } + + // Start the folder. This will cause a scan, should discover the other stuff in the folder + + m.StartFolder("ro") + m.ScanFolder("ro") + + // We should now have two files and two directories. + + size = m.GlobalSize("ro") + if size.Files != 2 || size.Directories != 2 { + t.Fatalf("Global: expected 2 files and 2 directories: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 2 || size.Directories != 2 { + t.Fatalf("Local: expected 2 files and 2 directories: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories == 0 { + t.Fatalf("ROChanged: expected something: %+v", size) + } + + // Revert should delete the unknown stuff + + m.Revert("ro") + + // These should still exist + for _, p := range []string{"_recvonly/knownDir/knownFile", "_recvonly/ignDir/ignFile"} { + _, err := os.Stat(p) + if err != nil { + t.Error("Unexpected error:", err) + } + } + + // These should have been removed + for _, p := range []string{"_recvonly/unknownDir", "_recvonly/unknownDir/unknownFile"} { + _, err := os.Stat(p) + if !os.IsNotExist(err) { + t.Error("Unexpected existing thing:", p) + } + } + + // We should now have one file and directory again. + + size = m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 files and 1 directories: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Local: expected 1 files and 1 directories: %+v", size) + } +} + +func TestRecvOnlyRevertNeeds(t *testing.T) { + // Make sure that a new file gets picked up and considered latest, then + // gets considered old when we hit Revert. + + os.RemoveAll("_recvonly") + defer os.RemoveAll("_recvonly") + + // Create some test data + + os.MkdirAll("_recvonly/.stfolder", 0755) + oldData := []byte("hello\n") + knownFiles := setupKnownFiles(t, oldData) + + // Get us a model up and running + + m := setupROFolder() + defer m.Stop() + + // Send and index update for the known stuff + + m.Index(device1, "ro", knownFiles) + m.updateLocalsFromScanning("ro", knownFiles) + + // Start the folder. This will cause a scan. + + m.StartFolder("ro") + m.ScanFolder("ro") + + // Everything should be in sync. + + size := m.GlobalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Global: expected 1 file and 1 directory: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Directories != 1 { + t.Fatalf("Local: expected 1 file and 1 directory: %+v", size) + } + size = m.NeedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("Need: expected nothing: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("ROChanged: expected nothing: %+v", size) + } + + // Update the file. + + newData := []byte("totally different data\n") + if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", newData, 0644); err != nil { + t.Fatal(err) + } + + // Rescan. + + if err := m.ScanFolder("ro"); err != nil { + t.Fatal(err) + } + + // We now have a newer file than the rest of the cluster. Global state should reflect this. + + size = m.GlobalSize("ro") + const sizeOfDir = 128 + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Global: expected the new file to be reflected: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Local: expected the new file to be reflected: %+v", size) + } + size = m.NeedSize("ro") + if size.Files+size.Directories > 0 { + t.Fatalf("Need: expected nothing: %+v", size) + } + size = m.ReceiveOnlyChangedSize("ro") + if size.Files+size.Directories == 0 { + t.Fatalf("ROChanged: expected something: %+v", size) + } + + // We hit the Revert button. The file that was new should become old. + + m.Revert("ro") + + size = m.GlobalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(oldData)) { + t.Fatalf("Global: expected the global size to revert: %+v", size) + } + size = m.LocalSize("ro") + if size.Files != 1 || size.Bytes != sizeOfDir+int64(len(newData)) { + t.Fatalf("Local: expected the local size to remain: %+v", size) + } + size = m.NeedSize("ro") + if size.Files != 1 || size.Bytes != int64(len(oldData)) { + t.Fatalf("Local: expected to need the old file data: %+v", size) + } +} + +func setupKnownFiles(t *testing.T, data []byte) []protocol.FileInfo { + if err := os.MkdirAll("_recvonly/knownDir", 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile("_recvonly/knownDir/knownFile", data, 0644); err != nil { + t.Fatal(err) + } + + t0 := time.Now().Add(-1 * time.Minute) + if err := os.Chtimes("_recvonly/knownDir/knownFile", t0, t0); err != nil { + t.Fatal(err) + } + + fi, err := os.Stat("_recvonly/knownDir/knownFile") + if err != nil { + t.Fatal(err) + } + knownFiles := []protocol.FileInfo{ + { + Name: "knownDir", + Type: protocol.FileInfoTypeDirectory, + Permissions: 0755, + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}}, + Sequence: 42, + }, + { + Name: "knownDir/knownFile", + Type: protocol.FileInfoTypeFile, + Permissions: 0644, + Size: fi.Size(), + ModifiedS: fi.ModTime().Unix(), + ModifiedNs: int32(fi.ModTime().UnixNano() % 1e9), + Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 42}}}, + Sequence: 42, + }, + } + + return knownFiles +} + +func setupROFolder() *Model { + fcfg := config.NewFolderConfiguration(protocol.LocalDeviceID, "ro", "receive only test", fs.FilesystemTypeBasic, "_recvonly") + fcfg.Type = config.FolderTypeReceiveOnly + fcfg.Devices = []config.FolderDeviceConfiguration{{DeviceID: device1}} + + cfg := defaultCfg.Copy() + cfg.Folders = append(cfg.Folders, fcfg) + + wrp := config.Wrap("/dev/null", cfg) + + db := db.OpenMemory() + m := NewModel(wrp, protocol.LocalDeviceID, "syncthing", "dev", db, nil) + + m.ServeBackground() + m.AddFolder(fcfg) + + return m +} diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index d3a7304ca..bd4a7ac7e 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -76,7 +76,7 @@ func (f *sendOnlyFolder) pull() bool { } file := intf.(protocol.FileInfo) - if !file.IsEquivalent(curFile, f.IgnorePerms, false) { + if !file.IsEquivalentOptional(curFile, f.IgnorePerms, false, 0) { return true } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 7635db9ad..998fb99b5 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -501,7 +501,11 @@ func (f *sendReceiveFolder) processDeletions(ignores *ignore.Matcher, fileDeleti } l.Debugln(f, "Deleting file", file.Name) - f.deleteFile(file, dbUpdateChan) + if update, err := f.deleteFile(file); err != nil { + f.newError("delete file", file.Name, err) + } else { + dbUpdateChan <- update + } } for i := range dirDeletions { @@ -736,7 +740,7 @@ func (f *sendReceiveFolder) handleDeleteDir(file protocol.FileInfo, ignores *ign } // deleteFile attempts to delete the given file -func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan<- dbUpdateJob) { +func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo) (dbUpdateJob, error) { // Used in the defer closure below, updated by the function body. Take // care not declare another err. var err error @@ -775,16 +779,18 @@ func (f *sendReceiveFolder) deleteFile(file protocol.FileInfo, dbUpdateChan chan if err == nil || fs.IsNotExist(err) { // It was removed or it doesn't exist to start with - dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} - } else if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { + return dbUpdateJob{file, dbUpdateDeleteFile}, nil + } + + if _, serr := f.fs.Lstat(file.Name); serr != nil && !fs.IsPermission(serr) { // We get an error just looking at the file, and it's not a permission // problem. Lets assume the error is in fact some variant of "file // does not exist" (possibly expressed as some parent being a file and // not a directory etc) and that the delete is handled. - dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile} - } else { - f.newError("delete file", file.Name, err) + return dbUpdateJob{file, dbUpdateDeleteFile}, nil } + + return dbUpdateJob{}, err } // renameFile attempts to rename an existing file to a destination @@ -1778,10 +1784,14 @@ func (f *sendReceiveFolder) deleteDir(dir string, ignores *ignore.Matcher, scanC } else if ignores != nil && ignores.Match(fullDirFile).IsIgnored() { hasIgnored = true } else if cf, ok := f.model.CurrentFolderFile(f.ID, fullDirFile); !ok || cf.IsDeleted() || cf.IsInvalid() { - // Something appeared in the dir that we either are not - // aware of at all, that we think should be deleted or that - // is invalid, but not currently ignored -> schedule scan - scanChan <- fullDirFile + // Something appeared in the dir that we either are not aware of + // at all, that we think should be deleted or that is invalid, + // but not currently ignored -> schedule scan. The scanChan + // might be nil, in which case we trust the scanning to be + // handled later as a result of our error return. + if scanChan != nil { + scanChan <- fullDirFile + } hasToBeScanned = true } else { // Dir contains file that is valid according to db and diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 1af9fb3f7..b3b007836 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -74,12 +74,12 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo { } } -func setUpModel(file protocol.FileInfo) *Model { +func setUpModel(files ...protocol.FileInfo) *Model { db := db.OpenMemory() model := NewModel(defaultCfgWrapper, protocol.LocalDeviceID, "syncthing", "dev", db, nil) model.AddFolder(defaultFolderConfig) // Update index - model.updateLocalsFromScanning("default", []protocol.FileInfo{file}) + model.updateLocalsFromScanning("default", files) return model } diff --git a/lib/model/model.go b/lib/model/model.go index 6ce39b1c8..e0b06ef18 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -57,6 +57,7 @@ const ( type service interface { BringToFront(string) Override(*db.FileSet, func([]protocol.FileInfo)) + Revert(*db.FileSet, func([]protocol.FileInfo)) DelayScan(d time.Duration) IgnoresUpdated() // ignore matcher was updated notification SchedulePull() // something relevant changed, we should try a pull @@ -690,6 +691,18 @@ func (m *Model) LocalSize(folder string) db.Counts { return db.Counts{} } +// ReceiveOnlyChangedSize returns the number of files, deleted files and +// total bytes for all files that have changed locally in a receieve only +// folder. +func (m *Model) ReceiveOnlyChangedSize(folder string) db.Counts { + m.fmut.RLock() + defer m.fmut.RUnlock() + if rf, ok := m.folderFiles[folder]; ok { + return rf.ReceiveOnlyChangedSize() + } + return db.Counts{} +} + // NeedSize returns the number and total size of currently needed files. func (m *Model) NeedSize(folder string) db.Counts { m.fmut.RLock() @@ -1747,6 +1760,12 @@ func sendIndexTo(prevSequence int64, conn protocol.Connection, folder string, fs // Mark the file as invalid if any of the local bad stuff flags are set. f.RawInvalid = f.IsInvalid() + // If the file is marked LocalReceive (i.e., changed locally on a + // receive only folder) we do not want it to ever become the + // globally best version, invalid or not. + if f.IsReceiveOnlyChanged() { + f.Version = protocol.Vector{} + } f.LocalFlags = 0 // never sent externally if dropSymlinks && f.IsSymlink() { @@ -1940,7 +1959,7 @@ func (m *Model) ScanFolderSubdirs(folder string, subs []string) error { return runner.Scan(subs) } -func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string) error { +func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, subDirs []string, localFlags uint32) error { m.fmut.RLock() if err := m.checkFolderRunningLocked(folder); err != nil { m.fmut.RUnlock() @@ -2010,6 +2029,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su ShortID: m.shortID, ProgressTickIntervalS: folderCfg.ScanProgressIntervalS, UseLargeBlocks: folderCfg.UseLargeBlocks, + LocalFlags: localFlags, }) if err := runner.CheckHealth(); err != nil { @@ -2106,6 +2126,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su ModifiedBy: m.id.Short(), Deleted: true, Version: f.Version.Update(m.shortID), + LocalFlags: localFlags, } // We do not want to override the global version // with the deleted file. Keeping only our local @@ -2289,6 +2310,24 @@ func (m *Model) Override(folder string) { }) } +func (m *Model) Revert(folder string) { + // Grab the runner and the file set. + + m.fmut.RLock() + fs, fsOK := m.folderFiles[folder] + runner, runnerOK := m.folderRunners[folder] + m.fmut.RUnlock() + if !fsOK || !runnerOK { + return + } + + // Run the revert, taking updates as if they came from scanning. + + runner.Revert(fs, func(files []protocol.FileInfo) { + m.updateLocalsFromScanning(folder, files) + }) +} + // CurrentSequence returns the change version for the given folder. // This is guaranteed to increment if the contents of the local folder has // changed. diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 16c64a058..11cfc7d83 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -49,6 +49,10 @@ func (f FileInfo) IsInvalid() bool { return f.RawInvalid || f.LocalFlags&LocalInvalidFlags != 0 } +func (f FileInfo) IsUnsupported() bool { + return f.LocalFlags&FlagLocalUnsupported != 0 +} + func (f FileInfo) IsIgnored() bool { return f.LocalFlags&FlagLocalIgnored != 0 } @@ -57,6 +61,10 @@ func (f FileInfo) MustRescan() bool { return f.LocalFlags&FlagLocalMustRescan != 0 } +func (f FileInfo) IsReceiveOnlyChanged() bool { + return f.LocalFlags&FlagLocalReceiveOnly != 0 +} + func (f FileInfo) IsDirectory() bool { return f.Type == FileInfoTypeDirectory } @@ -99,6 +107,10 @@ func (f FileInfo) FileName() string { return f.Name } +func (f FileInfo) FileLocalFlags() uint32 { + return f.LocalFlags +} + func (f FileInfo) ModTime() time.Time { return time.Unix(f.ModifiedS, int64(f.ModifiedNs)) } @@ -114,7 +126,7 @@ func (f FileInfo) FileVersion() Vector { // WinsConflict returns true if "f" is the one to choose when it is in // conflict with "other". func (f FileInfo) WinsConflict(other FileInfo) bool { - // If only one of the files is invalid, that one loses + // If only one of the files is invalid, that one loses. if f.IsInvalid() != other.IsInvalid() { return !f.IsInvalid() } @@ -145,7 +157,15 @@ func (f FileInfo) IsEmpty() bool { return f.Version.Counters == nil } -// IsEquivalent checks that the two file infos represent the same actual file content, +func (f FileInfo) IsEquivalent(other FileInfo) bool { + return f.isEquivalent(other, false, false, 0) +} + +func (f FileInfo) IsEquivalentOptional(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { + return f.isEquivalent(other, ignorePerms, ignoreBlocks, ignoreFlags) +} + +// isEquivalent checks that the two file infos represent the same actual file content, // i.e. it does purposely not check only selected (see below) struct members. // Permissions (config) and blocks (scanning) can be excluded from the comparison. // Any file info is not "equivalent", if it has different @@ -160,7 +180,7 @@ func (f FileInfo) IsEmpty() bool { // A symlink is not "equivalent", if it has different // - target // A directory does not have anything specific to check. -func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool) bool { +func (f FileInfo) isEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bool, ignoreFlags uint32) bool { if f.MustRescan() || other.MustRescan() { // These are per definition not equivalent because they don't // represent a valid state, even if both happen to have the @@ -168,6 +188,10 @@ func (f FileInfo) IsEquivalent(other FileInfo, ignorePerms bool, ignoreBlocks bo return false } + // Mask out the ignored local flags before checking IsInvalid() below + f.LocalFlags &^= ignoreFlags + other.LocalFlags &^= ignoreFlags + if f.Name != other.Name || f.Type != other.Type || f.Deleted != other.Deleted || f.IsInvalid() != other.IsInvalid() { return false } diff --git a/lib/protocol/deviceid.go b/lib/protocol/deviceid.go index c2a30b286..015080387 100644 --- a/lib/protocol/deviceid.go +++ b/lib/protocol/deviceid.go @@ -19,10 +19,18 @@ type DeviceID [DeviceIDLength]byte type ShortID uint64 var ( - LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} - EmptyDeviceID = DeviceID{ /* all zeroes */ } + LocalDeviceID = repeatedDeviceID(0xff) + GlobalDeviceID = repeatedDeviceID(0xf8) + EmptyDeviceID = DeviceID{ /* all zeroes */ } ) +func repeatedDeviceID(v byte) (d DeviceID) { + for i := range d { + d[i] = v + } + return +} + // NewDeviceID generates a new device ID from the raw bytes of a certificate func NewDeviceID(rawCert []byte) DeviceID { var n DeviceID diff --git a/lib/protocol/protocol.go b/lib/protocol/protocol.go index cdd20dd51..9db4316f3 100644 --- a/lib/protocol/protocol.go +++ b/lib/protocol/protocol.go @@ -94,14 +94,15 @@ const ( FlagLocalUnsupported = 1 << 0 // The kind is unsupported, e.g. symlinks on Windows FlagLocalIgnored = 1 << 1 // Matches local ignore patterns FlagLocalMustRescan = 1 << 2 // Doesn't match content on disk, must be rechecked fully + FlagLocalReceiveOnly = 1 << 3 // Change detected on receive only folder // Flags that should result in the Invalid bit on outgoing updates - LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan + LocalInvalidFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalMustRescan | FlagLocalReceiveOnly // Flags that should result in a file being in conflict with its // successor, due to us not having an up to date picture of its state on // disk. - LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored + LocalConflictFlags = FlagLocalUnsupported | FlagLocalIgnored | FlagLocalReceiveOnly ) var ( diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index 1ed48ab3c..2108a9733 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -423,6 +423,7 @@ func TestIsEquivalent(t *testing.T) { b FileInfo ignPerms *bool // nil means should not matter, we'll test both variants ignBlocks *bool + ignFlags uint32 eq bool } cases := []testCase{ @@ -491,6 +492,17 @@ func TestIsEquivalent(t *testing.T) { b: FileInfo{LocalFlags: FlagLocalUnsupported}, eq: true, }, + { + a: FileInfo{LocalFlags: 0}, + b: FileInfo{LocalFlags: FlagLocalReceiveOnly}, + eq: false, + }, + { + a: FileInfo{LocalFlags: 0}, + b: FileInfo{LocalFlags: FlagLocalReceiveOnly}, + ignFlags: FlagLocalReceiveOnly, + eq: true, + }, // Difference in blocks is not OK { @@ -588,10 +600,10 @@ func TestIsEquivalent(t *testing.T) { continue } - if res := tc.a.IsEquivalent(tc.b, ignPerms, ignBlocks); res != tc.eq { + if res := tc.a.isEquivalent(tc.b, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\na.IsEquivalent(b, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } - if res := tc.b.IsEquivalent(tc.a, ignPerms, ignBlocks); res != tc.eq { + if res := tc.b.isEquivalent(tc.a, ignPerms, ignBlocks, tc.ignFlags); res != tc.eq { t.Errorf("Case %d:\na: %v\nb: %v\nb.IsEquivalent(a, %v, %v) => %v, expected %v", i, tc.a, tc.b, ignPerms, ignBlocks, res, tc.eq) } } diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 37e2af4f7..76a55cb4a 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -68,6 +68,8 @@ type Config struct { ProgressTickIntervalS int // Whether to use large blocks for large files or the old standard of 128KiB for everything. UseLargeBlocks bool + // Local flags to set on scanned files + LocalFlags uint32 } type CurrentFiler interface { @@ -367,10 +369,11 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn ModifiedBy: w.ShortID, Size: info.Size(), RawBlockSize: int32(blockSize), + LocalFlags: w.LocalFlags, } if hasCurFile { - if curFile.IsEquivalent(f, w.IgnorePerms, true) { + if curFile.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if curFile.ShouldConflict() { @@ -407,10 +410,11 @@ func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, ModifiedS: info.ModTime().Unix(), ModifiedNs: int32(info.ModTime().Nanosecond()), ModifiedBy: w.ShortID, + LocalFlags: w.LocalFlags, } if ok { - if cf.IsEquivalent(f, w.IgnorePerms, true) { + if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if cf.ShouldConflict() { @@ -463,10 +467,11 @@ func (w *walker) walkSymlink(ctx context.Context, relPath string, dchan chan pro NoPermissions: true, // Symlinks don't have permissions of their own SymlinkTarget: target, ModifiedBy: w.ShortID, + LocalFlags: w.LocalFlags, } if ok { - if cf.IsEquivalent(f, w.IgnorePerms, true) { + if cf.IsEquivalentOptional(f, w.IgnorePerms, true, w.LocalFlags) { return nil } if cf.ShouldConflict() { diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index c4d70e79f..447d73655 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -221,8 +221,8 @@ func TestNormalization(t *testing.T) { // make sure it all gets done. In production, things will be correct // eventually... - walkDir(testFs, "normalization", nil, nil) - tmp := walkDir(testFs, "normalization", nil, nil) + walkDir(testFs, "normalization", nil, nil, 0) + tmp := walkDir(testFs, "normalization", nil, nil, 0) files := fileList(tmp).testfiles() @@ -267,7 +267,7 @@ func TestWalkSymlinkUnix(t *testing.T) { fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks") for _, path := range []string{".", "link"} { // Scan it - files := walkDir(fs, path, nil, nil) + files := walkDir(fs, path, nil, nil, 0) // Verify that we got one symlink and with the correct attributes if len(files) != 1 { @@ -300,7 +300,7 @@ func TestWalkSymlinkWindows(t *testing.T) { for _, path := range []string{".", "link"} { // Scan it - files := walkDir(fs, path, nil, nil) + files := walkDir(fs, path, nil, nil, 0) // Verify that we got zero symlinks if len(files) != 0 { @@ -329,7 +329,7 @@ func TestWalkRootSymlink(t *testing.T) { } // Scan it - files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil) + files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0) // Verify that we got two files if len(files) != 2 { @@ -353,7 +353,7 @@ func TestBlocksizeHysteresis(t *testing.T) { current := make(fakeCurrentFiler) runTest := func(expectedBlockSize int) { - files := walkDir(sf, ".", current, nil) + files := walkDir(sf, ".", current, nil, 0) if len(files) != 1 { t.Fatalf("expected one file, not %d", len(files)) } @@ -407,7 +407,57 @@ func TestBlocksizeHysteresis(t *testing.T) { runTest(512 << 10) } -func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher) []protocol.FileInfo { +func TestWalkReceiveOnly(t *testing.T) { + sf := fs.NewWalkFilesystem(&singleFileFS{ + name: "testfile.dat", + filesize: 1024, + }) + + current := make(fakeCurrentFiler) + + // Initial scan, no files in the CurrentFiler. Should pick up the file and + // set the ReceiveOnly flag on it, because that's the flag we give the + // walker to set. + + files := walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 1 { + t.Fatal("Should have scanned one file") + } + + if files[0].LocalFlags != protocol.FlagLocalReceiveOnly { + t.Fatal("Should have set the ReceiveOnly flag") + } + + // Update the CurrentFiler and scan again. It should not return + // anything, because the file has not changed. This verifies that the + // ReceiveOnly flag is properly ignored and doesn't trigger a rescan + // every time. + + cur := files[0] + current[cur.Name] = cur + + files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 0 { + t.Fatal("Should not have scanned anything") + } + + // Now pretend the file was previously ignored instead. We should pick up + // the difference in flags and set just the LocalReceive flags. + + cur.LocalFlags = protocol.FlagLocalIgnored + current[cur.Name] = cur + + files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly) + if len(files) != 1 { + t.Fatal("Should have scanned one file") + } + + if files[0].LocalFlags != protocol.FlagLocalReceiveOnly { + t.Fatal("Should have set the ReceiveOnly flag") + } +} + +func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo { fchan := Walk(context.TODO(), Config{ Filesystem: fs, Subs: []string{dir}, @@ -416,6 +466,7 @@ func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore. UseLargeBlocks: true, CurrentFiler: cfiler, Matcher: matcher, + LocalFlags: localFlags, }) var tmp []protocol.FileInfo @@ -579,7 +630,7 @@ func TestIssue4799(t *testing.T) { } fd.Close() - files := walkDir(fs, "/foo", nil, nil) + files := walkDir(fs, "/foo", nil, nil, 0) if len(files) != 1 || files[0].Name != "foo" { t.Error(`Received unexpected file infos when walking "/foo"`, files) } @@ -597,7 +648,7 @@ func TestRecurseInclude(t *testing.T) { t.Fatal(err) } - files := walkDir(testFs, ".", nil, ignores) + files := walkDir(testFs, ".", nil, ignores, 0) expected := []string{ filepath.Join("dir1"),