Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Refactor to use global list by version (fixes #6372) (#6638) 

Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors.

Co-authored-by: Jakob Borg <jakob@kastelo.net>
This commit is contained in:
Simon Frei 2020-05-30 09:50:23 +02:00 committed by GitHub
parent 1eea076f5c
commit 1f8e6c55f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1540 additions and 504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
cmd/stindex/idxck.go
View File

@ -219,10 +219,10 @@ func idxck(ldb backend.Backend) (success bool) {
fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
success = false
}
for i, fv := range vl.Versions {
dev, ok := deviceToIDs[string(fv.Device)]
checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
dev, ok := deviceToIDs[string(device)]
if !ok {
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
success = false
}
fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@ -235,14 +235,26 @@ func idxck(ldb backend.Backend) (success bool) {
if fi.VersionHash != nil {
fiv = versions[string(fi.VersionHash)]
}
if !fiv.Equal(fv.Version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
if !fiv.Equal(version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
success = false
}
if fi.IsInvalid() != fv.Invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
if fi.IsInvalid() != invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
success = false
}
if fi.IsDeleted() != deleted {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
success = false
}
}
for i, fv := range vl.RawVersions {
for _, device := range fv.Devices {
checkGlobal(i, device, fv.Version, false, fv.Deleted)
}
for _, device := range fv.InvalidDevices {
checkGlobal(i, device, fv.Version, true, fv.Deleted)
}
}
// If we need this file we should have a need entry for it. False
@ -251,7 +263,9 @@ func idxck(ldb backend.Backend) (success bool) {
if needsLocally(vl) {
_, ok := needs[gk]
if !ok {
dev := deviceToIDs[string(vl.Versions[0].Device)]
fv, _ := vl.GetGlobal()
devB, _ := fv.FirstDevice()
dev := deviceToIDs[string(devB)]
fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !fi.IsDeleted() && !fi.IsIgnored() {
fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
@ -319,15 +333,10 @@ func idxck(ldb backend.Backend) (success bool) {
}
func needsLocally(vl db.VersionList) bool {
var lv *protocol.Vector
for _, fv := range vl.Versions {
if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
lv = &fv.Version
break
}
}
if lv == nil {
fv, ok := vl.Get(protocol.LocalDeviceID[:])
if !ok {
return true // proviosinally, it looks like we need the file
}
return !lv.GreaterEqual(vl.Versions[0].Version)
gfv, _ := vl.GetGlobal() // Can't not have a global if we got something above
return !fv.Version.GreaterEqual(gfv.Version)
}

2
lib/api/api.go
View File

@ -1635,7 +1635,7 @@ func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) {
return json.Marshal(m)
}
func fileIntfJSONMap(f db.FileIntf) map[string]interface{} {
func fileIntfJSONMap(f protocol.FileIntf) map[string]interface{} {
out := map[string]interface{}{
"name": f.FileName(),
"type": f.FileType().String(),

14
lib/db/benchmark_test.go
View File

@ -187,7 +187,7 @@ func BenchmarkNeedHalf(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
count++
return true
})
@ -211,7 +211,7 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := fset.Snapshot()
snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
snap.WithNeed(remoteDevice0, func(fi protocol.FileIntf) bool {
count++
return true
})
@ -232,7 +232,7 @@ func BenchmarkHave(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
count++
return true
})
@ -253,7 +253,7 @@ func BenchmarkGlobal(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithGlobal(func(fi db.FileIntf) bool {
snap.WithGlobal(func(fi protocol.FileIntf) bool {
count++
return true
})
@ -274,7 +274,7 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
count++
return true
})
@ -295,7 +295,7 @@ func BenchmarkHaveTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
count++
return true
})
@ -316,7 +316,7 @@ func BenchmarkGlobalTruncated(b *testing.B) {
for i := 0; i < b.N; i++ {
count := 0
snap := benchS.Snapshot()
snap.WithGlobalTruncated(func(fi db.FileIntf) bool {
snap.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
count++
return true
})

57
lib/db/db_test.go
View File

@ -183,7 +183,9 @@ func TestUpdate0to3(t *testing.T) {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
}
key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid))
var key []byte
key, err = db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid))
if err != nil {
t.Fatal(err)
}
@ -201,7 +203,7 @@ func TestUpdate0to3(t *testing.T) {
t.Fatal(err)
}
defer trans.Release()
_ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool {
_ = trans.withHaveSequence(folder, 0, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
l.Infoln(f)
if found {
@ -228,12 +230,42 @@ func TestUpdate0to3(t *testing.T) {
haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
}
trans, err = db.newReadOnlyTransaction()
if err != nil {
t.Fatal(err)
}
defer trans.Release()
_ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
key, err = trans.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
t.Fatal(err)
}
dbi, err := trans.NewPrefixIterator(key)
if err != nil {
t.Fatal(err)
}
defer dbi.Release()
for dbi.Next() {
name := trans.keyer.NameFromGlobalVersionKey(dbi.Key())
key, err = trans.keyer.GenerateGlobalVersionKey(key, folder, name)
bs, err := trans.Get(key)
if err != nil {
t.Fatal(err)
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
t.Fatal(err)
}
key, err = trans.keyer.GenerateDeviceFileKey(key, folder, vl.Versions[0].Device, name)
fi, ok, err := trans.getFileTrunc(key, false)
if err != nil {
t.Fatal(err)
}
if !ok {
t.Fatal("surprise missing global file", string(name), protocol.DeviceIDFromBytes(vl.Versions[0].Device))
}
e, ok := need[fi.FileName()]
if !ok {
t.Error("Got unexpected needed file:", fi.FileName())
@ -243,8 +275,11 @@ func TestUpdate0to3(t *testing.T) {
if !f.IsEquivalentOptional(e, 0, true, true, 0) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
return true
})
}
if dbi.Error() != nil {
t.Fatal(err)
}
for n := range need {
t.Errorf(`Missing needed file "%v"`, n)
}
@ -467,7 +502,7 @@ func TestCheckGlobals(t *testing.T) {
}
// Clean up global entry of the now missing file
if err := db.checkGlobals([]byte(fs.folder), fs.meta); err != nil {
if err := db.checkGlobals([]byte(fs.folder)); err != nil {
t.Fatal(err)
}
@ -525,7 +560,7 @@ func TestUpdateTo10(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, v := range vl.Versions {
for _, v := range vl.RawVersions {
if !v.Deleted {
t.Error("Unexpected undeleted global version for a")
}
@ -535,10 +570,10 @@ func TestUpdateTo10(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !vl.Versions[0].Deleted {
if !vl.RawVersions[0].Deleted {
t.Error("vl.Versions[0] not deleted for b")
}
if vl.Versions[1].Deleted {
if vl.RawVersions[1].Deleted {
t.Error("vl.Versions[1] deleted for b")
}
// c
@ -546,10 +581,10 @@ func TestUpdateTo10(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if vl.Versions[0].Deleted {
if vl.RawVersions[0].Deleted {
t.Error("vl.Versions[0] deleted for c")
}
if !vl.Versions[1].Deleted {
if !vl.RawVersions[1].Deleted {
t.Error("vl.Versions[1] not deleted for c")
}
}

76
lib/db/lowlevel.go
View File

@ -426,7 +426,7 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke
return t.Commit()
}
func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
func (db *Lowlevel) checkGlobals(folder []byte) error {
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -444,9 +444,10 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
defer dbi.Release()
var dk []byte
ro := t.readOnlyTransaction
for dbi.Next() {
var vl VersionList
if err := vl.Unmarshal(dbi.Value()); err != nil || len(vl.Versions) == 0 {
if err := vl.Unmarshal(dbi.Value()); err != nil || vl.Empty() {
if err := t.Delete(dbi.Key()); err != nil {
return err
}
@ -459,36 +460,28 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
// we find those and clear them out.
name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
var newVL VersionList
for i, version := range vl.Versions {
dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, version.Device, name)
newVL := &VersionList{}
var changed, changedHere bool
for _, fv := range vl.RawVersions {
changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.Devices, newVL, ro)
if err != nil {
return err
}
_, err := t.Get(dk)
if backend.IsNotFound(err) {
continue
}
if err != nil {
return err
}
newVL.Versions = append(newVL.Versions, version)
changed = changed || changedHere
if i == 0 {
if fi, ok, err := t.getFileTrunc(dk, true); err != nil {
return err
} else if ok {
meta.addFile(protocol.GlobalDeviceID, fi)
}
changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.InvalidDevices, newVL, ro)
if err != nil {
return err
}
changed = changed || changedHere
}
if newLen := len(newVL.Versions); newLen == 0 {
if newVL.Empty() {
if err := t.Delete(dbi.Key()); err != nil {
return err
}
} else if newLen != len(vl.Versions) {
if err := t.Put(dbi.Key(), mustMarshal(&newVL)); err != nil {
} else if changed {
if err := t.Put(dbi.Key(), mustMarshal(newVL)); err != nil {
return err
}
}
@ -502,6 +495,30 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
return t.Commit()
}
func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *VersionList, t readOnlyTransaction) (bool, error) {
var changed bool
var err error
for _, device := range devices {
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, device, name)
if err != nil {
return false, err
}
f, ok, err := t.getFileTrunc(dk, true)
if err != nil {
return false, err
}
if !ok {
changed = true
continue
}
_, _, _, _, _, _, err = vl.update(folder, device, f, t)
if err != nil {
return false, err
}
}
return changed, nil
}
func (db *Lowlevel) getIndexID(device, folder []byte) (protocol.IndexID, error) {
key, err := db.keyer.GenerateIndexIDKey(nil, device, folder)
if err != nil {
@ -811,7 +828,7 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
meta := newMetadataTracker()
if err := db.checkGlobals([]byte(folder), meta); err != nil {
if err := db.checkGlobals([]byte(folder)); err != nil {
return nil, err
}
@ -831,8 +848,13 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
return nil, err
}
err = t.withGlobal([]byte(folder), nil, true, func(f protocol.FileIntf) bool {
meta.addFile(protocol.GlobalDeviceID, f)
return true
})
meta.emptyNeeded(protocol.LocalDeviceID)
err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f FileIntf) bool {
err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f protocol.FileIntf) bool {
meta.addNeeded(protocol.LocalDeviceID, f)
return true
})
@ -841,7 +863,7 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
}
for _, device := range meta.devices() {
meta.emptyNeeded(device)
err = t.withNeed([]byte(folder), device[:], true, func(f FileIntf) bool {
err = t.withNeed([]byte(folder), device[:], true, func(f protocol.FileIntf) bool {
meta.addNeeded(device, f)
return true
})
@ -878,7 +900,7 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
panic(err)
}
ok := true
if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi FileIntf) bool {
if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool {
ok = false // we got something, which we should not have
return false
}); err != nil && !backend.IsClosed(err) {
@ -1004,6 +1026,6 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
// unchanged checks if two files are the same and thus don't need to be updated.
// Local flags or the invalid bit might change without the version
// being bumped.
func unchanged(nf, ef FileIntf) bool {
func unchanged(nf, ef protocol.FileIntf) bool {
return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
}

14
lib/db/meta.go
View File

@ -147,7 +147,7 @@ func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts {
// addFile adds a file to the counts, adjusting the sequence number as
// appropriate
func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileIntf) {
m.mut.Lock()
defer m.mut.Unlock()
@ -186,7 +186,7 @@ func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
}
// addNeeded adds a file to the needed counts
func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f FileIntf) {
func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
m.mut.Lock()
defer m.mut.Unlock()
@ -201,7 +201,7 @@ func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 {
return m.countsPtr(dev, 0).Sequence
}
func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) {
func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileIntf) {
if dev == protocol.GlobalDeviceID {
return
}
@ -210,7 +210,7 @@ func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) {
}
}
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
cp := m.countsPtr(dev, flag)
switch {
@ -227,7 +227,7 @@ func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f Fi
}
// removeFile removes a file from the counts
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileIntf) {
if f.IsInvalid() && f.FileLocalFlags() == 0 {
// This is a remote invalid file; it does not count.
return
@ -250,7 +250,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
}
// removeNeeded removes a file from the needed counts
func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) {
func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
m.mut.Lock()
defer m.mut.Unlock()
@ -259,7 +259,7 @@ func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) {
m.removeFileLocked(dev, needFlag, f)
}
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
cp := m.countsPtr(dev, flag)
switch {

384
lib/db/schemaupdater.go
View File

@ -7,7 +7,10 @@
package db
import (
"bytes"
"errors"
"fmt"
"sort"
"strings"
"github.com/syncthing/syncthing/lib/db/backend"
@ -23,12 +26,14 @@ import (
// 7: v0.14.53
// 8-9: v1.4.0
// 10-11: v1.6.0
// 12: v1.7.0
// 12-13: v1.7.0
const (
dbVersion = 12
dbVersion = 13
dbMinSyncthingVersion = "v1.7.0"
)
var errFolderMissing = errors.New("folder present in global list but missing in keyer index")
type databaseDowngradeError struct {
minSyncthingVersion string
}
@ -89,14 +94,14 @@ func (db *schemaUpdater) updateSchema() error {
{9, db.updateSchemaTo9},
{10, db.updateSchemaTo10},
{11, db.updateSchemaTo11},
{12, db.updateSchemaTo12},
{13, db.updateSchemaTo13},
}
for _, m := range migrations {
if prevVersion < m.schemaVersion {
l.Infof("Migrating database to schema version %d...", m.schemaVersion)
if err := m.migration(int(prevVersion)); err != nil {
return err
return fmt.Errorf("failed migrating to version %v: %w", m.schemaVersion, err)
}
}
}
@ -128,8 +133,8 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
symlinkConv := 0
changedFolders := make(map[string]struct{})
ignAdded := 0
meta := newMetadataTracker() // dummy metadata tracker
var gk, buf []byte
var gk []byte
ro := t.readOnlyTransaction
for dbi.Next() {
folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
@ -155,17 +160,27 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
if err := t.Delete(dbi.Key()); err != nil {
return err
}
gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if err != nil {
return err
}
// Purposely pass nil file name to remove from global list,
// but don't touch meta and needs
buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil)
if err != nil && err != errEntryFromGlobalMissing {
fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
if backend.IsNotFound(err) {
// Shouldn't happen, but not critical.
continue
} else if err != nil {
return err
}
if err := t.Delete(dbi.Key()); err != nil {
_, _ = fl.pop(device)
if len(fl.Versions) == 0 {
err = t.Delete(gk)
} else {
err = t.Put(gk, mustMarshal(&fl))
}
if err != nil {
return err
}
continue
@ -199,13 +214,41 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
if err != nil {
return err
}
if buf, ok, err = t.updateGlobal(gk, buf, folder, device, f, meta); err != nil {
fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
if err != nil && !backend.IsNotFound(err) {
return err
} else if ok {
if _, ok = changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
i := 0
i = sort.Search(len(fl.Versions), func(j int) bool {
return fl.Versions[j].Invalid
})
for ; i < len(fl.Versions); i++ {
ordering := fl.Versions[i].Version.Compare(f.Version)
shouldInsert := ordering == protocol.Equal
if !shouldInsert {
shouldInsert, err = shouldInsertBefore(ordering, folder, fl.Versions[i].Device, true, f, ro)
if err != nil {
return err
}
}
ignAdded++
if shouldInsert {
nv := FileVersionDeprecated{
Device: device,
Version: f.Version,
Invalid: true,
}
fl.insertAt(i, nv)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
return err
}
if _, ok := changedFolders[string(folder)]; !ok {
changedFolders[string(folder)] = struct{}{}
}
ignAdded++
break
}
}
}
if err := t.Checkpoint(); err != nil {
@ -217,11 +260,6 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
return err
}
for folder := range changedFolders {
if err := db.dropFolderMeta([]byte(folder)); err != nil {
return err
}
}
return t.Commit()
}
@ -239,7 +277,7 @@ func (db *schemaUpdater) updateSchema1to2(_ int) error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f protocol.FileIntf) bool {
sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
if putErr != nil {
return false
@ -274,7 +312,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := t.withGlobal(folder, nil, true, func(f FileIntf) bool {
err := withGlobalBefore11(folder, true, func(f protocol.FileIntf) bool {
name := []byte(f.FileName())
dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
if putErr != nil {
@ -289,12 +327,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
if ok {
v = haveFile.FileVersion()
}
fv := FileVersion{
fv := FileVersionDeprecated{
Version: f.FileVersion(),
Invalid: f.IsInvalid(),
Deleted: f.IsDeleted(),
}
if !need(fv, ok, v) {
if !needDeprecated(fv, ok, v) {
return true
}
nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
@ -303,7 +341,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
}
putErr = t.Put(nk, nil)
return putErr == nil
})
}, t.readOnlyTransaction)
if putErr != nil {
return putErr
}
@ -359,7 +397,7 @@ func (db *schemaUpdater) updateSchema5to6(_ int) error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var iterErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f protocol.FileIntf) bool {
if !f.IsInvalid() {
return true
}
@ -404,7 +442,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var delErr error
err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
err := withNeedLocalBefore11(folder, false, func(f protocol.FileIntf) bool {
name := []byte(f.FileName())
gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
if delErr != nil {
@ -421,20 +459,20 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
delErr = t.Delete(key)
return delErr == nil
}
var fl VersionList
var fl VersionListDeprecated
err = fl.Unmarshal(svl)
if err != nil {
// This can't happen, but it's ignored everywhere else too,
// so lets not act on it.
return true
}
globalFV := FileVersion{
globalFV := FileVersionDeprecated{
Version: f.FileVersion(),
Invalid: f.IsInvalid(),
Deleted: f.IsDeleted(),
}
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) {
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !needDeprecated(globalFV, haveLocalFV, localFV.Version) {
key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
if err != nil {
delErr = err
@ -443,7 +481,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
delErr = t.Delete(key)
}
return delErr == nil
})
}, t.readOnlyTransaction)
if delErr != nil {
return delErr
}
@ -480,6 +518,7 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
if err != nil {
return err
}
defer it.Release()
for it.Next() {
intf, err := t.unmarshalTrunc(it.Value(), false)
if backend.IsNotFound(err) {
@ -510,6 +549,8 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
}
func (db *schemaUpdater) updateSchemaTo10(_ int) error {
// Rewrites global lists to include a Deleted flag.
t, err := db.newReadWriteTransaction()
if err != nil {
return err
@ -533,7 +574,7 @@ func (db *schemaUpdater) updateSchemaTo10(_ int) error {
defer dbi.Release()
for dbi.Next() {
var vl VersionList
var vl VersionListDeprecated
if err := vl.Unmarshal(dbi.Value()); err != nil {
return err
}
@ -592,7 +633,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
var putErr error
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi FileIntf) bool {
err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi protocol.FileIntf) bool {
f := fi.(FileInfoTruncated)
if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() || f.IsInvalid() || f.BlocksHash == nil {
return true
@ -620,7 +661,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
return t.Commit()
}
func (db *schemaUpdater) updateSchemaTo12(_ int) error {
func (db *schemaUpdater) updateSchemaTo13(prev int) error {
// Loads and rewrites all files, to deduplicate version vectors.
t, err := db.newReadWriteTransaction()
@ -629,9 +670,280 @@ func (db *schemaUpdater) updateSchemaTo12(_ int) error {
}
defer t.close()
if err := db.rewriteFiles(t); err != nil {
if prev < 12 {
if err := db.rewriteFiles(t); err != nil {
return err
}
}
if err := db.rewriteGlobals(t); err != nil {
return err
}
return t.Commit()
}
func (db *schemaUpdater) rewriteGlobals(t readWriteTransaction) error {
it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal})
if err != nil {
return err
}
defer it.Release()
for it.Next() {
var vl VersionListDeprecated
if err := vl.Unmarshal(it.Value()); err != nil {
// If we crashed during an earlier migration, some version
// lists might already be in the new format: Skip those.
var nvl VersionList
if nerr := nvl.Unmarshal(it.Value()); nerr == nil {
continue
}
return err
}
if len(vl.Versions) == 0 {
if err := t.Delete(it.Key()); err != nil {
return err
}
}
newVl, err := convertVersionList(vl)
if err != nil {
return err
}
if err := t.Put(it.Key(), mustMarshal(&newVl)); err != nil {
return err
}
if err := t.Checkpoint(); err != nil {
return err
}
}
it.Release()
return it.Error()
}
func convertVersionList(vl VersionListDeprecated) (VersionList, error) {
var newVl VersionList
var newPos, oldPos int
var lastVersion protocol.Vector
for _, fv := range vl.Versions {
if fv.Invalid {
break
}
oldPos++
if lastVersion.Equal(fv.Version) {
newVl.RawVersions[newPos].Devices = append(newVl.RawVersions[newPos].Devices, fv.Device)
continue
}
newPos = len(newVl.RawVersions)
newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, false, fv.Deleted))
lastVersion = fv.Version
}
if oldPos == len(vl.Versions) {
return newVl, nil
}
if len(newVl.RawVersions) == 0 {
fv := vl.Versions[oldPos]
newVl.RawVersions = []FileVersion{newFileVersion(fv.Device, fv.Version, true, fv.Deleted)}
}
newPos = 0
outer:
for _, fv := range vl.Versions[oldPos:] {
for _, nfv := range newVl.RawVersions[newPos:] {
switch nfv.Version.Compare(fv.Version) {
case protocol.Equal:
newVl.RawVersions[newPos].InvalidDevices = append(newVl.RawVersions[newPos].InvalidDevices, fv.Device)
lastVersion = fv.Version
continue outer
case protocol.Lesser:
newVl.insertAt(newPos, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
lastVersion = fv.Version
continue outer
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version is invalid, i.e. it looses anyway,
// no need to check/get the conflicting file.
}
newPos++
}
// Couldn't insert into any existing versions
newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
lastVersion = fv.Version
newPos++
}
return newVl, nil
}
func getGlobalVersionsByKeyBefore11(key []byte, t readOnlyTransaction) (VersionListDeprecated, error) {
bs, err := t.Get(key)
if err != nil {
return VersionListDeprecated{}, err
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
return VersionListDeprecated{}, err
}
return vl, nil
}
func withGlobalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key)
if err != nil {
return err
}
defer dbi.Release()
var dk []byte
for dbi.Next() {
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
var vl VersionListDeprecated
if err := vl.Unmarshal(dbi.Value()); err != nil {
return err
}
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
if err != nil {
return err
}
return dbi.Error()
}
func withNeedLocalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
if err != nil {
return err
}
dbi, err := t.NewPrefixIterator(key.WithoutName())
if err != nil {
return err
}
defer dbi.Release()
var keyBuf []byte
var f protocol.FileIntf
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = getGlobalBefore11(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate, t)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
}
return dbi.Error()
}
func getGlobalBefore11(keyBuf, folder, file []byte, truncate bool, t readOnlyTransaction) ([]byte, protocol.FileIntf, bool, error) {
keyBuf, err := t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
if err != nil {
return nil, nil, false, err
}
bs, err := t.Get(keyBuf)
if backend.IsNotFound(err) {
return keyBuf, nil, false, nil
} else if err != nil {
return nil, nil, false, err
}
var vl VersionListDeprecated
if err := vl.Unmarshal(bs); err != nil {
return nil, nil, false, err
}
if len(vl.Versions) == 0 {
return nil, nil, false, nil
}
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
if err != nil {
return nil, nil, false, err
}
fi, ok, err := t.getFileTrunc(keyBuf, truncate)
if err != nil || !ok {
return keyBuf, nil, false, err
}
return keyBuf, fi, true, nil
}
func (vl *VersionListDeprecated) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range vl.Versions {
if i > 0 {
b.WriteString(", ")
}
copy(id[:], v.Device)
fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
}
b.WriteString("}")
return b.String()
}
func (vl *VersionListDeprecated) pop(device []byte) (FileVersionDeprecated, int) {
for i, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
return v, i
}
}
return FileVersionDeprecated{}, -1
}
func (vl *VersionListDeprecated) Get(device []byte) (FileVersionDeprecated, bool) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
return v, true
}
}
return FileVersionDeprecated{}, false
}
func (vl *VersionListDeprecated) insertAt(i int, v FileVersionDeprecated) {
vl.Versions = append(vl.Versions, FileVersionDeprecated{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
}
func needDeprecated(global FileVersionDeprecated, haveLocal bool, localVersion protocol.Vector) bool {
// We never need an invalid file.
if global.Invalid {
return false
}
// We don't need a deleted file if we don't have it.
if global.Deleted && !haveLocal {
return false
}
// We don't need the global file if we already have the same version.
if haveLocal && localVersion.GreaterEqual(global.Version) {
return false
}
return true
}

35
lib/db/set.go
View File

@ -13,8 +13,6 @@
package db
import (
"time"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
@ -31,35 +29,10 @@ type FileSet struct {
updateMutex sync.Mutex // protects database updates and the corresponding metadata changes
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// FileInfoTruncated.
type FileIntf interface {
FileSize() int64
FileName() string
FileLocalFlags() uint32
IsDeleted() bool
IsInvalid() bool
IsIgnored() bool
IsUnsupported() bool
MustRescan() bool
IsReceiveOnlyChanged() bool
IsDirectory() bool
IsSymlink() bool
ShouldConflict() bool
HasPermissionBits() bool
SequenceNo() int64
BlockSize() int
FileVersion() protocol.Vector
FileType() protocol.FileInfoType
FilePermissions() uint32
FileModifiedBy() protocol.ShortID
ModTime() time.Time
}
// The Iterator is called with either a protocol.FileInfo or a
// FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
type Iterator func(f protocol.FileIntf) bool
func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
return &FileSet{
@ -335,7 +308,7 @@ func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated {
skip := (page - 1) * perpage
get := perpage
s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
s.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
if !f.IsReceiveOnlyChanged() {
return true
}
@ -359,7 +332,7 @@ func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage
files := make([]FileInfoTruncated, 0, perpage)
skip := (page - 1) * perpage
get := perpage
s.WithNeedTruncated(device, func(f FileIntf) bool {
s.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
if skip > 0 {
skip--
return true
@ -497,7 +470,7 @@ func normalizeFilenamesAndDropDuplicates(fs []protocol.FileInfo) []protocol.File
}
func nativeFileIterator(fn Iterator) Iterator {
return func(fi FileIntf) bool {
return func(fi protocol.FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name)

22
lib/db/set_test.go
View File

@ -48,7 +48,7 @@ func globalList(s *db.FileSet) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := s.Snapshot()
defer snap.Release()
snap.WithGlobal(func(fi db.FileIntf) bool {
snap.WithGlobal(func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -59,7 +59,7 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
var fs []db.FileInfoTruncated
snap := s.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
return true
@ -71,7 +71,7 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := s.Snapshot()
defer snap.Release()
snap.WithHave(n, func(fi db.FileIntf) bool {
snap.WithHave(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -83,7 +83,7 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi
var fs []db.FileInfoTruncated
snap := s.Snapshot()
defer snap.Release()
snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
fs = append(fs, f)
return true
@ -95,7 +95,7 @@ func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := s.Snapshot()
defer snap.Release()
snap.WithNeed(n, func(fi db.FileIntf) bool {
snap.WithNeed(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -998,7 +998,7 @@ func TestWithHaveSequence(t *testing.T) {
i := 2
snap := s.Snapshot()
defer snap.Release()
snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
snap.WithHaveSequence(int64(i), func(fi protocol.FileIntf) bool {
if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
}
@ -1049,7 +1049,7 @@ loop:
default:
}
snap := s.Snapshot()
snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileIntf) bool {
if fi.SequenceNo() < prevSeq+1 {
t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
}
@ -1527,8 +1527,8 @@ func TestSequenceIndex(t *testing.T) {
// Start a routine to walk the sequence index and inspect the result.
seen := make(map[string]db.FileIntf)
latest := make([]db.FileIntf, 0, len(local))
seen := make(map[string]protocol.FileIntf)
latest := make([]protocol.FileIntf, 0, len(local))
var seq int64
t0 := time.Now()
@ -1539,7 +1539,7 @@ func TestSequenceIndex(t *testing.T) {
// update has happened since our last iteration.
latest = latest[:0]
snap := s.Snapshot()
snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
snap.WithHaveSequence(seq+1, func(f protocol.FileIntf) bool {
seen[f.FileName()] = f
latest = append(latest, f)
seq = f.SequenceNo()
@ -1644,7 +1644,7 @@ func TestUpdateWithOneFileTwice(t *testing.T) {
snap := s.Snapshot()
defer snap.Release()
count := 0
snap.WithHaveSequence(0, func(f db.FileIntf) bool {
snap.WithHaveSequence(0, func(f protocol.FileIntf) bool {
count++
return true
})

358
lib/db/structs.go
View File

@ -12,7 +12,6 @@ package db
import (
"bytes"
"fmt"
"sort"
"time"
"github.com/syncthing/syncthing/lib/protocol"
@ -196,98 +195,317 @@ func (vl VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range vl.Versions {
for i, v := range vl.RawVersions {
if i > 0 {
b.WriteString(", ")
}
copy(id[:], v.Device)
fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
fmt.Fprintf(&b, "{%v, {", v.Version)
for j, dev := range v.Devices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
b.WriteString("}, {")
for j, dev := range v.InvalidDevices {
if j > 0 {
b.WriteString(", ")
}
copy(id[:], dev)
fmt.Fprint(&b, id.Short())
}
fmt.Fprint(&b, "}}")
}
b.WriteString("}")
return b.String()
}
// update brings the VersionList up to date with file. It returns the updated
// VersionList, a potentially removed old FileVersion and its index, as well as
// the index where the new FileVersion was inserted.
func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int, err error) {
vl, removedFV, removedAt = vl.pop(device)
nv := FileVersion{
Device: device,
Version: file.Version,
Invalid: file.IsInvalid(),
Deleted: file.IsDeleted(),
}
i := 0
if nv.Invalid {
i = sort.Search(len(vl.Versions), func(j int) bool {
return vl.Versions[j].Invalid
})
}
for ; i < len(vl.Versions); i++ {
switch vl.Versions[i].Version.Compare(file.Version) {
case protocol.Equal:
fallthrough
case protocol.Lesser:
// The version at this point in the list is equal to or lesser
// ("older") than us. We insert ourselves in front of it.
vl = vl.insertAt(i, nv)
return vl, removedFV, removedAt, i, nil
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version at this point is in conflict with us. We must pull
// the actual file metadata to determine who wins. If we win, we
// insert ourselves in front of the loser here. (The "Lesser" and
// "Greater" in the condition above is just based on the device
// IDs in the version vector, which is not the only thing we use
// to determine the winner.)
//
// A surprise missing file entry here is counted as a win for us.
if of, ok, err := t.getFile(folder, vl.Versions[i].Device, []byte(file.Name)); err != nil {
return vl, removedFV, removedAt, i, err
} else if !ok || file.WinsConflict(of) {
vl = vl.insertAt(i, nv)
return vl, removedFV, removedAt, i, nil
}
}
// VersionList, a device that has the global/newest version, a device that previously
// had the global/newest version, a boolean indicating if the global version has
// changed and if any error occurred (only possible in db interaction).
func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) {
if len(vl.RawVersions) == 0 {
nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
vl.RawVersions = append(vl.RawVersions, nv)
return nv, FileVersion{}, FileVersion{}, false, false, true, nil
}
// We didn't find a position for an insert above, so append to the end.
vl.Versions = append(vl.Versions, nv)
// Get the current global (before updating)
oldFV, haveOldGlobal := vl.GetGlobal()
return vl, removedFV, removedAt, len(vl.Versions) - 1, nil
// Remove ourselves first
removedFV, haveRemoved, _, err := vl.pop(folder, device, []byte(file.FileName()), t)
if err == nil {
// Find position and insert the file
err = vl.insert(folder, device, file, t)
}
if err != nil {
return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err
}
newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty
if !haveOldGlobal {
return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil
}
globalChanged := true
if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) {
globalChanged = false
}
return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
}
func (vl VersionList) insertAt(i int, v FileVersion) VersionList {
vl.Versions = append(vl.Versions, FileVersion{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
return vl
func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error {
var added bool
var err error
i := 0
for ; i < len(vl.RawVersions); i++ {
// Insert our new version
added, err = vl.checkInsertAt(i, folder, device, file, t)
if err != nil {
return err
}
if added {
break
}
}
if i == len(vl.RawVersions) {
// Append to the end
vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
}
return nil
}
func (vl *VersionList) insertAt(i int, v FileVersion) {
vl.RawVersions = append(vl.RawVersions, FileVersion{})
copy(vl.RawVersions[i+1:], vl.RawVersions[i:])
vl.RawVersions[i] = v
}
// pop returns the VersionList without the entry for the given device, as well
// as the removed FileVersion and the position, where that FileVersion was.
// If there is no FileVersion for the given device, the position is -1.
func (vl VersionList) pop(device []byte) (VersionList, FileVersion, int) {
for i, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
return vl, v, i
}
// as the removed FileVersion, whether it was found/removed at all and whether
// the global changed in the process.
func (vl *VersionList) pop(folder, device, name []byte, t readOnlyTransaction) (FileVersion, bool, bool, error) {
invDevice, i, j, ok := vl.findDevice(device)
if !ok {
return FileVersion{}, false, false, nil
}
return vl, FileVersion{}, -1
globalPos := vl.findGlobal()
if vl.RawVersions[i].deviceCount() == 1 {
fv := vl.RawVersions[i]
vl.popVersionAt(i)
return fv, true, globalPos == i, nil
}
if invDevice {
vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j)
} else {
vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j)
}
// If the last valid device of the previous global was removed above,
// the next entry is now the global entry (unless all entries are invalid).
if len(vl.RawVersions[i].Devices) == 0 && globalPos == i {
return vl.RawVersions[i], true, globalPos == vl.findGlobal(), nil
}
return vl.RawVersions[i], true, false, nil
}
func (vl VersionList) Get(device []byte) (FileVersion, bool) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
return v, true
// Get returns a FileVersion that contains the given device and whether it has
// been found at all.
func (vl *VersionList) Get(device []byte) (FileVersion, bool) {
_, i, _, ok := vl.findDevice(device)
if !ok {
return FileVersion{}, false
}
return vl.RawVersions[i], true
}
// GetGlobal returns the current global FileVersion. The returned FileVersion
// may be invalid, if all FileVersions are invalid. Returns false only if
// VersionList is empty.
func (vl *VersionList) GetGlobal() (FileVersion, bool) {
i := vl.findGlobal()
if i == -1 {
return FileVersion{}, false
}
return vl.RawVersions[i], true
}
func (vl *VersionList) Empty() bool {
return len(vl.RawVersions) == 0
}
// findGlobal returns the first version that isn't invalid, or if all versions are
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
func (vl *VersionList) findGlobal() int {
for i, fv := range vl.RawVersions {
if !fv.IsInvalid() {
return i
}
}
if len(vl.RawVersions) == 0 {
return -1
}
return 0
}
return FileVersion{}, false
// findDevices returns whether the device is in InvalidVersions or Versions and
// in InvalidDevices or Devices (true for invalid), the positions in the version
// and device slices and whether it has been found at all.
func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
for i, v := range vl.RawVersions {
if j := deviceIndex(v.Devices, device); j != -1 {
return false, i, j, true
}
if j := deviceIndex(v.InvalidDevices, device); j != -1 {
return true, i, j, true
}
}
return false, -1, -1, false
}
func (vl *VersionList) popVersion(version protocol.Vector) (FileVersion, bool) {
i := vl.versionIndex(version)
if i == -1 {
return FileVersion{}, false
}
fv := vl.RawVersions[i]
vl.popVersionAt(i)
return fv, true
}
func (vl *VersionList) versionIndex(version protocol.Vector) int {
for i, v := range vl.RawVersions {
if version.Equal(v.Version) {
return i
}
}
return -1
}
func (vl *VersionList) popVersionAt(i int) {
vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...)
}
// checkInsertAt determines if the given device and associated file should be
// inserted into the FileVersion at position i or into a new FileVersion at
// position i.
func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
ordering := vl.RawVersions[i].Version.Compare(file.FileVersion())
if ordering == protocol.Equal {
if !file.IsInvalid() {
vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device)
} else {
vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device)
}
return true, nil
}
existingDevice, _ := vl.RawVersions[i].FirstDevice()
insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t)
if err != nil {
return false, err
}
if insert {
vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
return true, nil
}
return false, nil
}
// shouldInsertBefore determines whether the file comes before an existing
// entry, given the version ordering (existing compared to new one), existing
// device and if the existing version is invalid.
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
switch ordering {
case protocol.Lesser:
// The version at this point in the list is lesser
// ("older") than us. We insert ourselves in front of it.
return true, nil
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version in conflict with us.
// Check if we can shortcut due to one being invalid.
if existingInvalid != file.IsInvalid() {
return existingInvalid, nil
}
// We must pull the actual file metadata to determine who wins.
// If we win, we insert ourselves in front of the loser here.
// (The "Lesser" and "Greater" in the condition above is just
// based on the device IDs in the version vector, which is not
// the only thing we use to determine the winner.)
of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
if err != nil {
return false, err
}
// A surprise missing file entry here is counted as a win for us.
if !ok {
return true, nil
}
if err != nil {
return false, err
}
if protocol.WinsConflict(file, of) {
return true, nil
}
}
return false, nil
}
func deviceIndex(devices [][]byte, device []byte) int {
for i, dev := range devices {
if bytes.Equal(device, dev) {
return i
}
}
return -1
}
func popDeviceAt(devices [][]byte, i int) [][]byte {
return append(devices[:i], devices[i+1:]...)
}
func popDevice(devices [][]byte, device []byte) ([][]byte, bool) {
i := deviceIndex(devices, device)
if i == -1 {
return devices, false
}
return popDeviceAt(devices, i), true
}
func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion {
fv := FileVersion{
Version: version,
Deleted: deleted,
}
if invalid {
fv.InvalidDevices = [][]byte{device}
} else {
fv.Devices = [][]byte{device}
}
return fv
}
func (fv FileVersion) FirstDevice() ([]byte, bool) {
if len(fv.Devices) != 0 {
return fv.Devices[0], true
}
if len(fv.InvalidDevices) != 0 {
return fv.InvalidDevices[0], true
}
return nil, false
}
func (fv FileVersion) IsInvalid() bool {
return len(fv.Devices) == 0
}
func (fv FileVersion) deviceCount() int {
return len(fv.Devices) + len(fv.InvalidDevices)
}
type fileList []protocol.FileInfo

715
lib/db/structs.pb.go
View File

@ -26,10 +26,10 @@ var _ = math.Inf
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type FileVersion struct {
Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
Invalid bool `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"`
Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
Deleted bool `protobuf:"varint,2,opt,name=deleted,proto3" json:"deleted,omitempty"`
Devices [][]byte `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"`
InvalidDevices [][]byte `protobuf:"bytes,4,rep,name=invalid_devices,json=invalidDevices,proto3" json:"invalid_devices,omitempty"`
}
func (m *FileVersion) Reset() { *m = FileVersion{} }
@ -66,7 +66,7 @@ func (m *FileVersion) XXX_DiscardUnknown() {
var xxx_messageInfo_FileVersion proto.InternalMessageInfo
type VersionList struct {
Versions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
RawVersions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
}
func (m *VersionList) Reset() { *m = VersionList{} }
@ -318,6 +318,82 @@ func (m *CountsSet) XXX_DiscardUnknown() {
var xxx_messageInfo_CountsSet proto.InternalMessageInfo
type FileVersionDeprecated struct {
Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
Invalid bool `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"`
Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
}
func (m *FileVersionDeprecated) Reset() { *m = FileVersionDeprecated{} }
func (m *FileVersionDeprecated) String() string { return proto.CompactTextString(m) }
func (*FileVersionDeprecated) ProtoMessage() {}
func (*FileVersionDeprecated) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{7}
}
func (m *FileVersionDeprecated) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *FileVersionDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_FileVersionDeprecated.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *FileVersionDeprecated) XXX_Merge(src proto.Message) {
xxx_messageInfo_FileVersionDeprecated.Merge(m, src)
}
func (m *FileVersionDeprecated) XXX_Size() int {
return m.ProtoSize()
}
func (m *FileVersionDeprecated) XXX_DiscardUnknown() {
xxx_messageInfo_FileVersionDeprecated.DiscardUnknown(m)
}
var xxx_messageInfo_FileVersionDeprecated proto.InternalMessageInfo
type VersionListDeprecated struct {
Versions []FileVersionDeprecated `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
}
func (m *VersionListDeprecated) Reset() { *m = VersionListDeprecated{} }
func (*VersionListDeprecated) ProtoMessage() {}
func (*VersionListDeprecated) Descriptor() ([]byte, []int) {
return fileDescriptor_e774e8f5f348d14d, []int{8}
}
func (m *VersionListDeprecated) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *VersionListDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_VersionListDeprecated.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *VersionListDeprecated) XXX_Merge(src proto.Message) {
xxx_messageInfo_VersionListDeprecated.Merge(m, src)
}
func (m *VersionListDeprecated) XXX_Size() int {
return m.ProtoSize()
}
func (m *VersionListDeprecated) XXX_DiscardUnknown() {
xxx_messageInfo_VersionListDeprecated.DiscardUnknown(m)
}
var xxx_messageInfo_VersionListDeprecated proto.InternalMessageInfo
func init() {
proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
proto.RegisterType((*VersionList)(nil), "db.VersionList")
@ -326,61 +402,68 @@ func init() {
proto.RegisterType((*IndirectionHashesOnly)(nil), "db.IndirectionHashesOnly")
proto.RegisterType((*Counts)(nil), "db.Counts")
proto.RegisterType((*CountsSet)(nil), "db.CountsSet")
proto.RegisterType((*FileVersionDeprecated)(nil), "db.FileVersionDeprecated")
proto.RegisterType((*VersionListDeprecated)(nil), "db.VersionListDeprecated")
}
func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) }
var fileDescriptor_e774e8f5f348d14d = []byte{
// 774 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x8f, 0xe3, 0x44,
0x10, 0x8d, 0x37, 0x71, 0x3e, 0xca, 0x49, 0xd8, 0x6d, 0x96, 0x91, 0x15, 0x09, 0xc7, 0x0a, 0x5a,
0xc9, 0xe2, 0x90, 0xc0, 0xec, 0x0d, 0x24, 0x0e, 0x61, 0x35, 0x22, 0x12, 0x62, 0x51, 0x67, 0xb5,
0xa7, 0x95, 0x22, 0x7f, 0x74, 0x92, 0xd6, 0x38, 0xee, 0xe0, 0xee, 0xcc, 0xc8, 0xf3, 0x17, 0xb8,
0x70, 0xe4, 0x38, 0x17, 0xfe, 0xcb, 0x1c, 0xe7, 0x88, 0x38, 0x44, 0x90, 0x70, 0x80, 0x7f, 0x81,
0xba, 0xdb, 0x76, 0x3c, 0x73, 0x61, 0x6e, 0x55, 0xaf, 0x2a, 0xa9, 0xaa, 0xf7, 0x9e, 0x1b, 0x7a,
0x5c, 0xa4, 0xbb, 0x50, 0xf0, 0xf1, 0x36, 0x65, 0x82, 0xa1, 0x67, 0x51, 0x30, 0xf8, 0x2c, 0x25,
0x5b, 0xc6, 0x27, 0x0a, 0x08, 0x76, 0xcb, 0xc9, 0x8a, 0xad, 0x98, 0x4a, 0x54, 0xa4, 0x1b, 0x07,
0x67, 0x31, 0x0d, 0x74, 0x4b, 0xc8, 0xe2, 0x49, 0x40, 0xb6, 0x1a, 0x1f, 0xfd, 0x6c, 0x80, 0x75,
0x41, 0x63, 0xf2, 0x9e, 0xa4, 0x9c, 0xb2, 0x04, 0x7d, 0x01, 0xad, 0x2b, 0x1d, 0xda, 0x86, 0x6b,
0x78, 0xd6, 0xf9, 0xf3, 0x71, 0xf1, 0xab, 0xf1, 0x7b, 0x12, 0x0a, 0x96, 0x4e, 0x1b, 0x77, 0xfb,
0x61, 0x0d, 0x17, 0x6d, 0xe8, 0x0c, 0x9a, 0x11, 0xb9, 0xa2, 0x21, 0xb1, 0x9f, 0xb9, 0x86, 0xd7,
0xc5, 0x79, 0x86, 0x6c, 0x68, 0xd1, 0xe4, 0xca, 0x8f, 0x69, 0x64, 0xd7, 0x5d, 0xc3, 0x6b, 0xe3,
0x22, 0x95, 0x95, 0x88, 0xc4, 0x44, 0x90, 0xc8, 0x6e, 0xe8, 0x4a, 0x9e, 0x8e, 0x2e, 0xc0, 0xca,
0x17, 0xf9, 0x9e, 0x72, 0x81, 0xbe, 0x84, 0x76, 0x3e, 0x85, 0xdb, 0x86, 0x5b, 0xf7, 0xac, 0xf3,
0x8f, 0xc6, 0x51, 0x30, 0xae, 0xec, 0x9b, 0x2f, 0x53, 0xb6, 0x7d, 0xd5, 0xf8, 0xf5, 0x76, 0x58,
0x1b, 0xfd, 0x66, 0xc2, 0x0b, 0xd9, 0x35, 0x4b, 0x96, 0xec, 0x5d, 0xba, 0x4b, 0x42, 0x5f, 0x90,
0x08, 0x21, 0x68, 0x24, 0xfe, 0x86, 0xa8, 0xc3, 0x3a, 0x58, 0xc5, 0x12, 0xe3, 0xf4, 0x86, 0xa8,
0x15, 0xeb, 0x58, 0xc5, 0xe8, 0x53, 0x80, 0x0d, 0x8b, 0xe8, 0x92, 0x92, 0x68, 0xc1, 0x6d, 0x53,
0x55, 0x3a, 0x05, 0x32, 0x47, 0x1f, 0xc0, 0x2a, 0xcb, 0x41, 0x66, 0x77, 0x5d, 0xc3, 0x6b, 0x4c,
0xbf, 0x96, 0x7b, 0xfc, 0xb1, 0x1f, 0xbe, 0x5e, 0x51, 0xb1, 0xde, 0x05, 0xe3, 0x90, 0x6d, 0x26,
0x3c, 0x4b, 0x42, 0xb1, 0xa6, 0xc9, 0xaa, 0x12, 0x55, 0x65, 0x18, 0xcf, 0xd7, 0x2c, 0x15, 0xb3,
0x37, 0xb8, 0x1c, 0x37, 0xcd, 0xaa, 0x02, 0x74, 0x9e, 0x26, 0xc0, 0x00, 0xda, 0x9c, 0xfc, 0xb4,
0x23, 0x49, 0x48, 0x6c, 0x50, 0xcb, 0x96, 0x39, 0x7a, 0x05, 0x7d, 0x9e, 0x6d, 0x62, 0x9a, 0x5c,
0x2e, 0x84, 0x9f, 0xae, 0x88, 0xb0, 0x5f, 0xa8, 0xe3, 0x7b, 0x39, 0xfa, 0x4e, 0x81, 0x68, 0x08,
0x56, 0x10, 0xb3, 0xf0, 0x92, 0x2f, 0xd6, 0x3e, 0x5f, 0xdb, 0x48, 0x09, 0x09, 0x1a, 0xfa, 0xce,
0xe7, 0x6b, 0xf4, 0x39, 0x34, 0x44, 0xb6, 0xd5, 0x12, 0xf7, 0xcf, 0xcf, 0x4e, 0x2b, 0x95, 0x2c,
0x67, 0x5b, 0x82, 0x55, 0x0f, 0x72, 0xc1, 0xda, 0x92, 0x74, 0x43, 0xb9, 0x16, 0x4e, 0x4a, 0xdc,
0xc3, 0x55, 0x48, 0x8e, 0x2b, 0x19, 0x4c, 0xb8, 0x6d, 0xb9, 0x86, 0x67, 0x9e, 0x48, 0xf8, 0x81,
0xa3, 0x09, 0xe8, 0xe1, 0x0b, 0xa5, 0x4d, 0x4f, 0xd6, 0xa7, 0xcf, 0x0f, 0xfb, 0x61, 0x17, 0xfb,
0xd7, 0x53, 0x59, 0x98, 0xd3, 0x1b, 0x82, 0x3b, 0x41, 0x11, 0xca, 0x99, 0x31, 0x0b, 0xfd, 0x78,
0xb1, 0x8c, 0xfd, 0x15, 0xb7, 0xff, 0x69, 0xa9, 0xa1, 0xa0, 0xb0, 0x0b, 0x09, 0xa1, 0x11, 0x74,
0x73, 0xc2, 0xf4, 0x8d, 0xff, 0xb6, 0xd4, 0x91, 0x56, 0x0e, 0xaa, 0x2b, 0x2b, 0xc6, 0x6c, 0x3e,
0x30, 0x26, 0xf2, 0x4e, 0x66, 0x96, 0xbf, 0x6b, 0x4f, 0xfb, 0x87, 0xfd, 0x10, 0xb0, 0x7f, 0x3d,
0xd3, 0xe8, 0xc9, 0xdc, 0xaf, 0xa0, 0x9f, 0xb0, 0x45, 0x95, 0x80, 0xb6, 0xfa, 0xab, 0x5e, 0xc2,
0x7e, 0x3c, 0x81, 0xb9, 0x4f, 0xbf, 0x81, 0x8e, 0x3a, 0x27, 0x77, 0x7b, 0x53, 0x25, 0x85, 0xd7,
0x3f, 0x3e, 0xb1, 0xac, 0x70, 0x49, 0x73, 0xae, 0x7d, 0xde, 0x38, 0xfa, 0x00, 0x9f, 0xcc, 0x92,
0x88, 0xa6, 0x24, 0x14, 0xf9, 0x0d, 0x84, 0xbf, 0x4d, 0xe2, 0xec, 0xff, 0x05, 0x7d, 0x02, 0x1d,
0xa3, 0xbf, 0x0d, 0x68, 0x7e, 0xcb, 0x76, 0x89, 0xe0, 0xe8, 0x25, 0x98, 0x4b, 0x1a, 0x13, 0xae,
0xbe, 0x1d, 0x13, 0xeb, 0x44, 0xb2, 0xae, 0x87, 0xb3, 0x94, 0x12, 0xae, 0xcc, 0x61, 0xe2, 0x2a,
0xa4, 0xbc, 0xa9, 0x9d, 0xc6, 0xd5, 0x27, 0x66, 0xe2, 0x32, 0x7f, 0xfc, 0x0c, 0x98, 0x27, 0xb6,
0x5f, 0x82, 0x19, 0x64, 0x82, 0x14, 0xdf, 0x9e, 0x4e, 0x1e, 0xf8, 0xbc, 0xf9, 0xc8, 0xe7, 0x03,
0x68, 0xeb, 0x67, 0x67, 0xf6, 0x46, 0x39, 0xbc, 0x8b, 0xcb, 0x1c, 0x39, 0x50, 0xf1, 0x81, 0xa2,
0xe2, 0x81, 0x33, 0x46, 0x6f, 0xa1, 0xa3, 0xaf, 0x9c, 0x13, 0x81, 0x3c, 0x68, 0x86, 0x2a, 0xc9,
0x45, 0x00, 0xf9, 0xe0, 0xe8, 0x72, 0xc1, 0xbd, 0xae, 0xcb, 0xf5, 0xc3, 0x94, 0xc8, 0x87, 0x45,
0x1d, 0x5e, 0xc7, 0x45, 0x3a, 0x75, 0xef, 0xfe, 0x72, 0x6a, 0x77, 0x07, 0xc7, 0xb8, 0x3f, 0x38,
0xc6, 0x9f, 0x07, 0xa7, 0xf6, 0xcb, 0xd1, 0xa9, 0xdd, 0x1e, 0x1d, 0xe3, 0xfe, 0xe8, 0xd4, 0x7e,
0x3f, 0x3a, 0xb5, 0xa0, 0xa9, 0x94, 0x7d, 0xfd, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3a, 0x45,
0x1c, 0xc5, 0xce, 0x05, 0x00, 0x00,
// 861 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4f, 0x8f, 0xdb, 0x54,
0x10, 0x8f, 0x9b, 0xff, 0xe3, 0x64, 0xdb, 0xbe, 0x76, 0x57, 0x66, 0x25, 0x1c, 0xcb, 0x08, 0x61,
0x71, 0x48, 0x60, 0x7b, 0xa3, 0x12, 0x42, 0x61, 0x55, 0x11, 0x09, 0x51, 0xf4, 0xb6, 0xf4, 0x80,
0x2a, 0x45, 0xb6, 0xf3, 0x92, 0x3c, 0xd5, 0xf1, 0x0b, 0x7e, 0xce, 0xae, 0xdc, 0x4f, 0xc1, 0x05,
0x89, 0x03, 0x87, 0x5e, 0xf8, 0x2e, 0x7b, 0xec, 0x11, 0x71, 0x88, 0x20, 0xcb, 0x01, 0xbe, 0x05,
0x7a, 0xf3, 0x6c, 0xc7, 0x1b, 0x0e, 0xb4, 0xb7, 0x99, 0xdf, 0xcc, 0xf3, 0xcc, 0xfc, 0xe6, 0xe7,
0x81, 0xbe, 0x4c, 0x93, 0x4d, 0x98, 0xca, 0xe1, 0x3a, 0x11, 0xa9, 0x20, 0x77, 0x66, 0xc1, 0xe9,
0x07, 0x09, 0x5b, 0x0b, 0x39, 0x42, 0x20, 0xd8, 0xcc, 0x47, 0x0b, 0xb1, 0x10, 0xe8, 0xa0, 0xa5,
0x13, 0x4f, 0x4f, 0x22, 0x1e, 0xe8, 0x94, 0x50, 0x44, 0xa3, 0x80, 0xad, 0x35, 0xee, 0xfe, 0x62,
0x80, 0xf9, 0x84, 0x47, 0xec, 0x39, 0x4b, 0x24, 0x17, 0x31, 0xf9, 0x04, 0xda, 0x97, 0xda, 0xb4,
0x0c, 0xc7, 0xf0, 0xcc, 0xb3, 0x7b, 0xc3, 0xe2, 0xd5, 0xf0, 0x39, 0x0b, 0x53, 0x91, 0x8c, 0x1b,
0xd7, 0xdb, 0x41, 0x8d, 0x16, 0x69, 0xc4, 0x82, 0xf6, 0x8c, 0x45, 0x2c, 0x65, 0x33, 0xeb, 0x8e,
0x63, 0x78, 0x1d, 0x5a, 0xb8, 0x3a, 0x72, 0xc9, 0x43, 0x26, 0xad, 0xba, 0x53, 0xf7, 0x7a, 0xb4,
0x70, 0xc9, 0x47, 0x70, 0x97, 0xc7, 0x97, 0x7e, 0xc4, 0x67, 0xd3, 0x22, 0xa3, 0x81, 0x19, 0x47,
0x39, 0x7c, 0xae, 0x51, 0xf7, 0x3b, 0x30, 0xf3, 0xce, 0xbe, 0xe6, 0x32, 0x25, 0x5f, 0x40, 0x27,
0x2f, 0x2b, 0x2d, 0xc3, 0xa9, 0x7b, 0xe6, 0xd9, 0xdd, 0xe1, 0x2c, 0x18, 0x56, 0x06, 0x18, 0x3f,
0x50, 0xdd, 0xed, 0xb6, 0x03, 0x93, 0xfa, 0x57, 0x39, 0x26, 0x69, 0xf9, 0xea, 0xb3, 0xc6, 0xcf,
0xaf, 0x07, 0x35, 0xf7, 0xd7, 0x26, 0xdc, 0x57, 0x8f, 0x26, 0xf1, 0x5c, 0x3c, 0x4b, 0x36, 0x71,
0xe8, 0xab, 0x7e, 0x09, 0x34, 0x62, 0x7f, 0xc5, 0x70, 0xf0, 0x2e, 0x45, 0x5b, 0x61, 0x92, 0xbf,
0x62, 0x56, 0xdd, 0x31, 0xbc, 0x3a, 0x45, 0x9b, 0xbc, 0x0f, 0xb0, 0x12, 0x33, 0x3e, 0xe7, 0x6c,
0x36, 0x95, 0x56, 0x13, 0x23, 0xdd, 0x02, 0xb9, 0x20, 0x2f, 0xc0, 0x2c, 0xc3, 0x41, 0x66, 0xf5,
0x1c, 0xc3, 0x6b, 0x8c, 0x1f, 0xab, 0xb6, 0x7e, 0xdf, 0x0e, 0x1e, 0x2d, 0x78, 0xba, 0xdc, 0x04,
0xc3, 0x50, 0xac, 0x46, 0x32, 0x8b, 0xc3, 0x74, 0xc9, 0xe3, 0x45, 0xc5, 0xaa, 0xae, 0x69, 0x78,
0xb1, 0x14, 0x49, 0x3a, 0x39, 0xa7, 0x65, 0xb9, 0x71, 0x56, 0x5d, 0x50, 0xf7, 0xed, 0x16, 0x74,
0x0a, 0x1d, 0xc9, 0x7e, 0xd8, 0xb0, 0x38, 0x64, 0x16, 0x60, 0xb3, 0xa5, 0x4f, 0x3e, 0x84, 0x23,
0x99, 0xad, 0x22, 0x1e, 0xbf, 0x9c, 0xa6, 0x7e, 0xb2, 0x60, 0xa9, 0x75, 0x1f, 0x87, 0xef, 0xe7,
0xe8, 0x33, 0x04, 0xc9, 0x00, 0xcc, 0x20, 0x12, 0xe1, 0x4b, 0x39, 0x5d, 0xfa, 0x72, 0x69, 0x11,
0xc7, 0xf0, 0x7a, 0x14, 0x34, 0xf4, 0x95, 0x2f, 0x97, 0xe4, 0x63, 0x68, 0xa4, 0xd9, 0x9a, 0xa1,
0x02, 0x8e, 0xce, 0x4e, 0xf6, 0x2d, 0x95, 0x2c, 0x67, 0x6b, 0x46, 0x31, 0x87, 0x38, 0x60, 0xae,
0x59, 0xb2, 0xe2, 0x52, 0xef, 0xb1, 0xe1, 0x18, 0x5e, 0x9f, 0x56, 0x21, 0x55, 0xae, 0x64, 0x30,
0x96, 0x96, 0xe9, 0x18, 0x5e, 0x73, 0x4f, 0xc2, 0x37, 0x92, 0x8c, 0x40, 0x17, 0x9f, 0xe2, 0x6e,
0xfa, 0x2a, 0x3e, 0xbe, 0xb7, 0xdb, 0x0e, 0x7a, 0xd4, 0xbf, 0x1a, 0xab, 0xc0, 0x05, 0x7f, 0xc5,
0x68, 0x37, 0x28, 0x4c, 0x55, 0x33, 0x12, 0xa1, 0x1f, 0x4d, 0xe7, 0x91, 0xbf, 0x90, 0xd6, 0xdf,
0x6d, 0x2c, 0x0a, 0x88, 0x3d, 0x51, 0x10, 0x71, 0xa1, 0x97, 0x13, 0xa6, 0x67, 0xfc, 0xa7, 0x8d,
0x43, 0x9a, 0x39, 0x88, 0x53, 0x56, 0xa4, 0xde, 0xba, 0x2d, 0x75, 0x0f, 0xda, 0xb9, 0x72, 0x2d,
0xf5, 0xae, 0x33, 0x3e, 0xda, 0x6d, 0x07, 0x40, 0xfd, 0xab, 0x89, 0x46, 0x69, 0x11, 0x56, 0x8c,
0xc7, 0x62, 0x5a, 0x25, 0xa0, 0x83, 0x9f, 0xea, 0xc7, 0xe2, 0xdb, 0x3d, 0x98, 0xeb, 0xf4, 0x73,
0xe8, 0xe2, 0x38, 0x28, 0xfe, 0x4f, 0xa1, 0x85, 0x4e, 0x21, 0xfd, 0x07, 0x7b, 0x96, 0x11, 0x57,
0x34, 0xe7, 0xbb, 0xcf, 0x13, 0xdd, 0x17, 0x70, 0x3c, 0x89, 0x67, 0x3c, 0x61, 0x61, 0x9a, 0xcf,
0xc0, 0xe4, 0xd3, 0x38, 0xca, 0xfe, 0x7f, 0xa1, 0x6f, 0x41, 0x87, 0xfb, 0x97, 0x01, 0xad, 0x2f,
0xc5, 0x26, 0x4e, 0x25, 0x79, 0x08, 0xcd, 0x39, 0x8f, 0x98, 0xc4, 0x7f, 0xa7, 0x49, 0xb5, 0xa3,
0x58, 0xd7, 0xc5, 0x45, 0xc2, 0x99, 0x44, 0x71, 0x34, 0x69, 0x15, 0x42, 0x6d, 0x6a, 0xa5, 0x49,
0xfc, 0xc5, 0x9a, 0xb4, 0xf4, 0xab, 0x6c, 0x37, 0x30, 0x54, 0xb2, 0xfd, 0x10, 0x9a, 0x41, 0x96,
0xb2, 0xe2, 0xdf, 0xd3, 0xce, 0x2d, 0x9d, 0xb7, 0x0e, 0x74, 0x7e, 0x0a, 0x1d, 0x7d, 0x68, 0x26,
0xe7, 0xa8, 0xf0, 0x1e, 0x2d, 0x7d, 0x62, 0x43, 0x45, 0x07, 0x48, 0xc5, 0x2d, 0x65, 0xb8, 0x4f,
0xa1, 0xab, 0xa7, 0xbc, 0x60, 0x29, 0xf1, 0xa0, 0x15, 0xa2, 0x93, 0x2f, 0x01, 0xd4, 0xfd, 0xd1,
0xe1, 0x82, 0x7b, 0x1d, 0x57, 0xed, 0x87, 0x09, 0xf3, 0x8b, 0xbb, 0x58, 0xa7, 0x85, 0xeb, 0xfe,
0x64, 0xc0, 0x71, 0xe5, 0x64, 0x9d, 0xb3, 0x75, 0xc2, 0xf4, 0x05, 0x7a, 0xf7, 0xeb, 0x7b, 0x02,
0x2d, 0x3d, 0x08, 0x16, 0xe9, 0xd1, 0xdc, 0x53, 0xd5, 0x0b, 0x41, 0xd6, 0xb5, 0x54, 0x0b, 0x01,
0x1e, 0xd0, 0xba, 0x17, 0xb1, 0xfb, 0x3d, 0x1c, 0x57, 0x8e, 0x6d, 0xa5, 0xad, 0xc7, 0xff, 0x39,
0xbb, 0xef, 0x1d, 0x9c, 0xdd, 0x7d, 0x72, 0xde, 0xe0, 0xc1, 0xc5, 0x1d, 0x3b, 0xd7, 0x7f, 0xda,
0xb5, 0xeb, 0x9d, 0x6d, 0xbc, 0xd9, 0xd9, 0xc6, 0x1f, 0x3b, 0xbb, 0xf6, 0xe3, 0x8d, 0x5d, 0x7b,
0x7d, 0x63, 0x1b, 0x6f, 0x6e, 0xec, 0xda, 0x6f, 0x37, 0x76, 0x2d, 0x68, 0xe1, 0xa4, 0x8f, 0xfe,
0x0d, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xe1, 0xbd, 0x08, 0xe2, 0x06, 0x00, 0x00,
}
func (m *FileVersion) Marshal() (dAtA []byte, err error) {
@ -403,6 +486,24 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.InvalidDevices) > 0 {
for iNdEx := len(m.InvalidDevices) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.InvalidDevices[iNdEx])
copy(dAtA[i:], m.InvalidDevices[iNdEx])
i = encodeVarintStructs(dAtA, i, uint64(len(m.InvalidDevices[iNdEx])))
i--
dAtA[i] = 0x22
}
}
if len(m.Devices) > 0 {
for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- {
i -= len(m.Devices[iNdEx])
copy(dAtA[i:], m.Devices[iNdEx])
i = encodeVarintStructs(dAtA, i, uint64(len(m.Devices[iNdEx])))
i--
dAtA[i] = 0x1a
}
}
if m.Deleted {
i--
if m.Deleted {
@ -411,24 +512,7 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) {
dAtA[i] = 0
}
i--
dAtA[i] = 0x20
}
if m.Invalid {
i--
if m.Invalid {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i--
dAtA[i] = 0x18
}
if len(m.Device) > 0 {
i -= len(m.Device)
copy(dAtA[i:], m.Device)
i = encodeVarintStructs(dAtA, i, uint64(len(m.Device)))
i--
dAtA[i] = 0x12
dAtA[i] = 0x10
}
{
size, err := m.Version.MarshalToSizedBuffer(dAtA[:i])
@ -463,10 +547,10 @@ func (m *VersionList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.Versions) > 0 {
for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- {
if len(m.RawVersions) > 0 {
for iNdEx := len(m.RawVersions) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
size, err := m.RawVersions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
@ -813,6 +897,103 @@ func (m *CountsSet) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *FileVersionDeprecated) Marshal() (dAtA []byte, err error) {
size := m.ProtoSize()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *FileVersionDeprecated) MarshalTo(dAtA []byte) (int, error) {
size := m.ProtoSize()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *FileVersionDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Deleted {
i--
if m.Deleted {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i--
dAtA[i] = 0x20
}
if m.Invalid {
i--
if m.Invalid {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i--
dAtA[i] = 0x18
}
if len(m.Device) > 0 {
i -= len(m.Device)
copy(dAtA[i:], m.Device)
i = encodeVarintStructs(dAtA, i, uint64(len(m.Device)))
i--
dAtA[i] = 0x12
}
{
size, err := m.Version.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintStructs(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
return len(dAtA) - i, nil
}
func (m *VersionListDeprecated) Marshal() (dAtA []byte, err error) {
size := m.ProtoSize()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *VersionListDeprecated) MarshalTo(dAtA []byte) (int, error) {
size := m.ProtoSize()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *VersionListDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.Versions) > 0 {
for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintStructs(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
}
}
return len(dAtA) - i, nil
}
func encodeVarintStructs(dAtA []byte, offset int, v uint64) int {
offset -= sovStructs(v)
base := offset
@ -832,16 +1013,21 @@ func (m *FileVersion) ProtoSize() (n int) {
_ = l
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
l = len(m.Device)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
if m.Invalid {
n += 2
}
if m.Deleted {
n += 2
}
if len(m.Devices) > 0 {
for _, b := range m.Devices {
l = len(b)
n += 1 + l + sovStructs(uint64(l))
}
}
if len(m.InvalidDevices) > 0 {
for _, b := range m.InvalidDevices {
l = len(b)
n += 1 + l + sovStructs(uint64(l))
}
}
return n
}
@ -851,8 +1037,8 @@ func (m *VersionList) ProtoSize() (n int) {
}
var l int
_ = l
if len(m.Versions) > 0 {
for _, e := range m.Versions {
if len(m.RawVersions) > 0 {
for _, e := range m.RawVersions {
l = e.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
}
@ -1007,6 +1193,42 @@ func (m *CountsSet) ProtoSize() (n int) {
return n
}
func (m *FileVersionDeprecated) ProtoSize() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = m.Version.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
l = len(m.Device)
if l > 0 {
n += 1 + l + sovStructs(uint64(l))
}
if m.Invalid {
n += 2
}
if m.Deleted {
n += 2
}
return n
}
func (m *VersionListDeprecated) ProtoSize() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if len(m.Versions) > 0 {
for _, e := range m.Versions {
l = e.ProtoSize()
n += 1 + l + sovStructs(uint64(l))
}
}
return n
}
func sovStructs(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
@ -1076,8 +1298,28 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
}
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.Deleted = bool(v != 0)
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
@ -1104,36 +1346,14 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...)
if m.Device == nil {
m.Device = []byte{}
}
m.Devices = append(m.Devices, make([]byte, postIndex-iNdEx))
copy(m.Devices[len(m.Devices)-1], dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.Invalid = bool(v != 0)
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field InvalidDevices", wireType)
}
var v int
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
@ -1143,12 +1363,24 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.Deleted = bool(v != 0)
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.InvalidDevices = append(m.InvalidDevices, make([]byte, postIndex-iNdEx))
copy(m.InvalidDevices[len(m.InvalidDevices)-1], dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
@ -1204,7 +1436,7 @@ func (m *VersionList) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field RawVersions", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -1231,8 +1463,8 @@ func (m *VersionList) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Versions = append(m.Versions, FileVersion{})
if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
m.RawVersions = append(m.RawVersions, FileVersion{})
if err := m.RawVersions[len(m.RawVersions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -2243,6 +2475,253 @@ func (m *CountsSet) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *FileVersionDeprecated) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: FileVersionDeprecated: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: FileVersionDeprecated: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.Version.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...)
if m.Device == nil {
m.Device = []byte{}
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.Invalid = bool(v != 0)
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.Deleted = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *VersionListDeprecated) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: VersionListDeprecated: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: VersionListDeprecated: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStructs
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStructs
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthStructs
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Versions = append(m.Versions, FileVersionDeprecated{})
if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStructs(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthStructs
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipStructs(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0

22
lib/db/structs.proto
View File

@ -13,15 +13,15 @@ option (gogoproto.goproto_unrecognized_all) = false;
option (gogoproto.goproto_sizecache_all) = false;
message FileVersion {
protocol.Vector version = 1 [(gogoproto.nullable) = false];
bytes device = 2;
bool invalid = 3;
bool deleted = 4;
protocol.Vector version = 1 [(gogoproto.nullable) = false];
bool deleted = 2;
repeated bytes devices = 3;
repeated bytes invalid_devices = 4;
}
message VersionList {
option (gogoproto.goproto_stringer) = false;
repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
repeated FileVersion versions = 1 [(gogoproto.customname) = "RawVersions", (gogoproto.nullable) = false];
}
// Must be the same as FileInfo but without the blocks field
@ -79,3 +79,15 @@ message CountsSet {
repeated Counts counts = 1 [(gogoproto.nullable) = false];
int64 created = 2; // unix nanos
}
message FileVersionDeprecated {
protocol.Vector version = 1 [(gogoproto.nullable) = false];
bytes device = 2;
bool invalid = 3;
bool deleted = 4;
}
message VersionListDeprecated {
option (gogoproto.goproto_stringer) = false;
repeated FileVersionDeprecated versions = 1 [(gogoproto.nullable) = false];
}

227
lib/db/transactions.go
View File

@ -15,7 +15,11 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
var errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry")
var (
errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry")
errEmptyGlobal = errors.New("no versions in global list")
errEmptyFileVersion = errors.New("no devices in global file version")
)
// A readOnlyTransaction represents a database snapshot.
type readOnlyTransaction struct {
@ -54,7 +58,7 @@ func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool,
return f.(protocol.FileInfo), true, nil
}
func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, bool, error) {
func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileIntf, bool, error) {
bs, err := t.Get(key)
if backend.IsNotFound(err) {
return nil, false, nil
@ -72,7 +76,7 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo
return f, true, nil
}
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) {
func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileIntf, error) {
if trunc {
var tf FileInfoTruncated
err := tf.Unmarshal(bs)
@ -175,26 +179,44 @@ func (t readOnlyTransaction) getGlobalVersionsByKey(key []byte) (VersionList, er
return vl, nil
}
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) {
func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, protocol.FileIntf, bool, error) {
vl, err := t.getGlobalVersions(keyBuf, folder, file)
if backend.IsNotFound(err) {
return keyBuf, nil, false, nil
} else if err != nil {
return nil, nil, false, err
}
if len(vl.Versions) == 0 {
return nil, nil, false, nil
}
var fi protocol.FileIntf
keyBuf, fi, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, truncate, vl)
return keyBuf, fi, true, err
}
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
func (t readOnlyTransaction) getGlobalFromVersionList(keyBuf, folder, file []byte, truncate bool, vl VersionList) ([]byte, protocol.FileIntf, FileVersion, error) {
fv, ok := vl.GetGlobal()
if !ok {
return keyBuf, nil, FileVersion{}, errEmptyGlobal
}
keyBuf, fi, err := t.getGlobalFromFileVersion(keyBuf, folder, file, truncate, fv)
return keyBuf, fi, fv, err
}
func (t readOnlyTransaction) getGlobalFromFileVersion(keyBuf, folder, file []byte, truncate bool, fv FileVersion) ([]byte, protocol.FileIntf, error) {
dev, ok := fv.FirstDevice()
if !ok {
return keyBuf, nil, errEmptyFileVersion
}
keyBuf, err := t.keyer.GenerateDeviceFileKey(keyBuf, folder, dev, file)
if err != nil {
return nil, nil, false, err
return keyBuf, nil, err
}
fi, ok, err := t.getFileTrunc(keyBuf, truncate)
if err != nil || !ok {
return keyBuf, nil, false, err
if err != nil {
return keyBuf, nil, err
}
return keyBuf, fi, true, nil
if !ok {
return keyBuf, nil, errEntryFromGlobalMissing
}
return keyBuf, fi, nil
}
func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
@ -320,19 +342,12 @@ func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, f
return err
}
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
var f protocol.FileIntf
dk, f, _, err = t.getGlobalFromVersionList(dk, folder, name, truncate, vl)
if err != nil {
return err
}
f, ok, err := t.getFileTrunc(dk, truncate)
if err != nil {
return err
}
if !ok {
continue
}
if !fn(f) {
return nil
}
@ -393,16 +408,13 @@ func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.Devi
return nil, err
}
var devices []protocol.DeviceID
for _, v := range vl.Versions {
if !v.Version.Equal(vl.Versions[0].Version) {
break
}
if v.Invalid {
continue
}
n := protocol.DeviceIDFromBytes(v.Device)
devices = append(devices, n)
fv, ok := vl.GetGlobal()
if !ok {
return nil, nil
}
devices := make([]protocol.DeviceID, len(fv.Devices))
for i, dev := range fv.Devices {
devices[i] = protocol.DeviceIDFromBytes(dev)
}
return devices, nil
@ -431,25 +443,28 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
return err
}
globalFV := vl.Versions[0]
globalFV, ok := vl.GetGlobal()
if !ok {
return errEmptyGlobal
}
haveFV, have := vl.Get(device)
if !need(globalFV, have, haveFV.Version) {
continue
}
name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, globalFV.Device, name)
if err != nil {
return err
}
gf, ok, err := t.getFileTrunc(dk, truncate)
var gf protocol.FileIntf
dk, gf, err = t.getGlobalFromFileVersion(dk, folder, name, truncate, globalFV)
if err != nil {
return err
}
globalDev, ok := globalFV.FirstDevice()
if !ok {
return errEntryFromGlobalMissing
return errEmptyFileVersion
}
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, globalFV.Version, globalFV.Device)
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalDev)
if !fn(gf) {
return dbi.Error()
}
@ -469,7 +484,7 @@ func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Ite
defer dbi.Release()
var keyBuf []byte
var f FileIntf
var f protocol.FileIntf
var ok bool
for dbi.Next() {
keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
@ -586,7 +601,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
return nil, false, err
}
fl, removedFV, removedAt, insertedAt, err := fl.update(folder, device, file, t.readOnlyTransaction)
globalFV, oldGlobalFV, removedFV, haveOldGlobal, haveRemoved, globalChanged, err := fl.update(folder, device, file, t.readOnlyTransaction)
if err != nil {
return nil, false, err
}
@ -601,26 +616,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
// Only load those from db if actually needed
var gotGlobal, gotOldGlobal bool
var global, oldGlobal FileIntf
globalFV := fl.Versions[0]
var oldGlobalFV FileVersion
haveOldGlobal := false
globalUnaffected := removedAt != 0 && insertedAt != 0
if globalUnaffected {
oldGlobalFV = globalFV
haveOldGlobal = true
} else {
if removedAt == 0 {
oldGlobalFV = removedFV
haveOldGlobal = true
} else if len(fl.Versions) > 1 {
// The previous newest version is now at index 1
oldGlobalFV = fl.Versions[1]
haveOldGlobal = true
}
}
var global, oldGlobal protocol.FileIntf
// Check the need of the device that was updated
// Must happen before updating global meta: If this is the first
@ -628,11 +624,11 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
needBefore := false
if haveOldGlobal {
needBefore = need(oldGlobalFV, removedAt >= 0, removedFV.Version)
needBefore = need(oldGlobalFV, haveRemoved, removedFV.Version)
}
needNow := need(globalFV, true, fl.Versions[insertedAt].Version)
needNow := need(globalFV, true, file.Version)
if needBefore {
if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil {
return nil, false, err
}
gotOldGlobal = true
@ -644,7 +640,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
}
}
if needNow {
if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil {
return nil, false, err
}
gotGlobal = true
@ -657,42 +653,34 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
}
// Update global size counter if necessary
// Necessary here means the first item in the global list was changed,
// even if both new and old are invalid, due to potential change in
// LocalFlags.
// Neither the global state nor the needs of any devices, except the one
// updated, changed.
if globalUnaffected {
if !globalChanged {
// Neither the global state nor the needs of any devices, except
// the one updated, changed.
return keyBuf, true, nil
}
// Remove the old global from the global size counter
if haveOldGlobal {
if !gotOldGlobal {
if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil {
return nil, false, err
}
gotOldGlobal = true
}
// Remove the old global from the global size counter
meta.removeFile(protocol.GlobalDeviceID, oldGlobal)
}
// Add the new global to the global size counter
if !gotGlobal {
if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil {
return nil, false, err
}
gotGlobal = true
}
meta.addFile(protocol.GlobalDeviceID, global)
// If global changed, but both the new and old are invalid, noone needed
// the file before and now -> nothing to do.
if global.IsInvalid() && (!haveOldGlobal || oldGlobal.IsInvalid()) {
return keyBuf, true, nil
}
// check for local (if not already done before)
if !bytes.Equal(device, protocol.LocalDeviceID[:]) {
localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:])
@ -736,40 +724,12 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
return keyBuf, true, nil
}
func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, insertedAt int, fl VersionList) (FileIntf, error) {
if insertedAt == 0 {
func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, fv FileVersion) ([]byte, protocol.FileIntf, error) {
if fv.Version.Equal(file.Version) {
// Inserted a new newest version
return file, nil
return keyBuf, file, nil
}
var err error
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name)
if err != nil {
return nil, err
}
global, ok, err := t.getFileTrunc(keyBuf, true)
if err != nil {
return nil, err
}
if !ok {
return nil, errEntryFromGlobalMissing
}
return global, nil
}
func (t readWriteTransaction) updateGlobalGetOldGlobal(keyBuf, folder, name []byte, oldGlobalFV FileVersion) (FileIntf, error) {
var err error
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name)
if err != nil {
return nil, err
}
oldGlobal, ok, err := t.getFileTrunc(keyBuf, true)
if err != nil {
return nil, err
}
if !ok {
return nil, errEntryFromGlobalMissing
}
return oldGlobal, nil
return t.getGlobalFromFileVersion(keyBuf, folder, name, true, fv)
}
func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add bool) ([]byte, error) {
@ -790,7 +750,7 @@ func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add b
func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
// We never need an invalid file.
if global.Invalid {
if global.IsInvalid() {
return false
}
// We don't need a deleted file if we don't have it.
@ -807,7 +767,7 @@ func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool
// removeFromGlobal removes the device from the global version list for the
// given file. If the version list is empty after this, the file entry is
// removed entirely.
func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) {
func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file []byte, meta *metadataTracker) ([]byte, error) {
deviceID := protocol.DeviceIDFromBytes(device)
l.Debugf("remove from global; folder=%q device=%v file=%q", folder, deviceID, file)
@ -821,34 +781,32 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
return nil, err
}
if len(fl.Versions) == 0 {
oldGlobalFV, haveOldGlobal := fl.GetGlobal()
if !haveOldGlobal {
// Shouldn't ever happen, but doesn't hurt to handle.
return keyBuf, t.Delete(gk)
}
oldGlobalFV := fl.Versions[0]
fl, removedFV, removedAt := fl.pop(device)
if removedAt == -1 {
removedFV, haveRemoved, globalChanged, err := fl.pop(folder, device, file, t.readOnlyTransaction)
if err != nil {
return nil, err
}
if !haveRemoved {
// There is no version for the given device
return keyBuf, nil
}
var global FileIntf
var global protocol.FileIntf
var gotGlobal, ok bool
globalFV, ok := fl.GetGlobal()
// Add potential needs of the removed device
if len(fl.Versions) != 0 && !fl.Versions[0].Invalid && need(fl.Versions[0], false, protocol.Vector{}) && !need(oldGlobalFV, removedAt != -1, removedFV.Version) {
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
if ok && !globalFV.IsInvalid() && need(globalFV, false, protocol.Vector{}) && !need(oldGlobalFV, haveRemoved, removedFV.Version) {
keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl)
if err != nil {
return nil, err
}
global, ok, err = t.getFileTrunc(keyBuf, true)
if err != nil {
return nil, err
} else if !ok {
return nil, errEntryFromGlobalMissing
}
gotGlobal = true
meta.addNeeded(deviceID, global)
if bytes.Equal(protocol.LocalDeviceID[:], device) {
@ -859,7 +817,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
}
// Global hasn't changed, abort early
if removedAt != 0 {
if !globalChanged {
l.Debugf("new global after remove: %v", fl)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
return nil, err
@ -896,7 +854,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
}
// Nothing left, i.e. nothing to add to the global counter below.
if len(fl.Versions) == 0 {
if fl.Empty() {
if err := t.Delete(gk); err != nil {
return nil, err
}
@ -905,21 +863,14 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
// Add to global
if !gotGlobal {
keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl)
if err != nil {
return nil, err
}
global, ok, err = t.getFileTrunc(keyBuf, true)
if err != nil {
return nil, err
}
if !ok {
return nil, errEntryFromGlobalMissing
}
}
meta.addFile(protocol.GlobalDeviceID, global)
l.Debugf("new global after remove: %v", fl)
l.Debugf(`new global for "%s" after remove: %v`, file, fl)
if err := t.Put(gk, mustMarshal(&fl)); err != nil {
return nil, err
}

6
lib/model/folder.go
View File

@ -303,7 +303,7 @@ func (f *folder) pull() (success bool) {
// If there is nothing to do, don't even enter sync-waiting state.
abort := true
snap := f.fset.Snapshot()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
abort = false
return false
})
@ -499,7 +499,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
for _, sub := range subDirs {
var iterError error
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi protocol.FileIntf) bool {
select {
case <-f.ctx.Done():
return false
@ -634,7 +634,7 @@ func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file proto
found := false
nf := protocol.FileInfo{}
snap.WithBlocksHash(file.BlocksHash, func(ifi db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(ifi protocol.FileIntf) bool {
fi := ifi.(protocol.FileInfo)
select {

2
lib/model/folder_recvonly.go
View File

@ -87,7 +87,7 @@ func (f *receiveOnlyFolder) revert() {
batchSizeBytes := 0
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
fi := intf.(protocol.FileInfo)
if !fi.IsReceiveOnlyChanged() {
// We're only interested in files that have changed locally in

4
lib/model/folder_sendonly.go
View File

@ -52,7 +52,7 @@ func (f *sendOnlyFolder) pull() bool {
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromPulling(batch)
batch = batch[:0]
@ -110,7 +110,7 @@ func (f *sendOnlyFolder) override() {
batchSizeBytes := 0
snap := f.fset.Snapshot()
defer snap.Release()
snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
f.updateLocalsFromScanning(batch)

2
lib/model/folder_sendrecv.go
View File

@ -305,7 +305,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
// Regular files to pull goes into the file queue, everything else
// (directories, symlinks and deletes) goes into the "process directly"
// pile.
snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
select {
case <-f.ctx.Done():
return false

6
lib/model/model.go
View File

@ -851,7 +851,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
}
rest = make([]db.FileInfoTruncated, 0, perpage)
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
if cfg.IgnoreDelete && f.IsDeleted() {
return true
}
@ -1936,7 +1936,7 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
snap := s.fset.Snapshot()
defer snap.Release()
previousWasDelete := false
snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
snap.WithHaveSequence(s.prevSequence+1, func(fi protocol.FileIntf) bool {
// This is to make sure that renames (which is an add followed by a delete) land in the same batch.
// Even if the batch is full, we allow a last delete to slip in, we do this by making sure that
// the batch ends with a non-delete, or that the last item in the batch is already a delete
@ -2248,7 +2248,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
snap := files.Snapshot()
defer snap.Release()
snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
// Don't include the prefix itself.

22
lib/model/model_test.go
View File

@ -2426,7 +2426,7 @@ func TestIssue3496(t *testing.T) {
m.fmut.RUnlock()
var localFiles []protocol.FileInfo
snap := fs.Snapshot()
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
localFiles = append(localFiles, i.(protocol.FileInfo))
return true
})
@ -3556,7 +3556,7 @@ func TestRenameSequenceOrder(t *testing.T) {
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
count++
return true
})
@ -3588,7 +3588,7 @@ func TestRenameSequenceOrder(t *testing.T) {
var firstExpectedSequence int64
var secondExpectedSequence int64
failed := false
snap.WithHaveSequence(0, func(i db.FileIntf) bool {
snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
t.Log(i)
if i.FileName() == "17" {
firstExpectedSequence = i.SequenceNo() + 1
@ -3621,7 +3621,7 @@ func TestRenameSameFile(t *testing.T) {
count := 0
snap := dbSnapshot(t, m, "default")
snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
count++
return true
})
@ -3644,7 +3644,7 @@ func TestRenameSameFile(t *testing.T) {
prevSeq := int64(0)
seen := false
snap.WithHaveSequence(0, func(i db.FileIntf) bool {
snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
if i.SequenceNo() <= prevSeq {
t.Fatalf("non-increasing sequences: %d <= %d", i.SequenceNo(), prevSeq)
}
@ -3683,7 +3683,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count := 0
snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@ -3693,7 +3693,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@ -3712,7 +3712,7 @@ func TestRenameEmptyFile(t *testing.T) {
defer snap.Release()
count = 0
snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
count++
return true
})
@ -3722,7 +3722,7 @@ func TestRenameEmptyFile(t *testing.T) {
}
count = 0
snap.WithBlocksHash(file.BlocksHash, func(i db.FileIntf) bool {
snap.WithBlocksHash(file.BlocksHash, func(i protocol.FileIntf) bool {
count++
if i.FileName() != "new-file" {
t.Fatalf("unexpected file name %s, expected new-file", i.FileName())
@ -3757,7 +3757,7 @@ func TestBlockListMap(t *testing.T) {
}
var paths []string
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})
@ -3790,7 +3790,7 @@ func TestBlockListMap(t *testing.T) {
defer snap.Release()
paths = paths[:0]
snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
paths = append(paths, fi.FileName())
return true
})

29
lib/protocol/bep_extensions.go
View File

@ -23,6 +23,31 @@ const (
Version13HelloMagic uint32 = 0x9F79BC40 // old
)
// FileIntf is the set of methods implemented by both FileInfo and
// db.FileInfoTruncated.
type FileIntf interface {
FileSize() int64
FileName() string
FileLocalFlags() uint32
IsDeleted() bool
IsInvalid() bool
IsIgnored() bool
IsUnsupported() bool
MustRescan() bool
IsReceiveOnlyChanged() bool
IsDirectory() bool
IsSymlink() bool
ShouldConflict() bool
HasPermissionBits() bool
SequenceNo() int64
BlockSize() int
FileVersion() Vector
FileType() FileInfoType
FilePermissions() uint32
FileModifiedBy() ShortID
ModTime() time.Time
}
func (m Hello) Magic() uint32 {
return HelloMessageMagic
}
@ -139,7 +164,7 @@ func (f FileInfo) FileModifiedBy() ShortID {
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {
func WinsConflict(f, other FileIntf) bool {
// If only one of the files is invalid, that one loses.
if f.IsInvalid() != other.IsInvalid() {
return !f.IsInvalid()
@ -164,7 +189,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool {
// The modification times were equal. Use the device ID in the version
// vector as tie breaker.
return f.Version.Compare(other.Version) == ConcurrentGreater
return f.FileVersion().Compare(other.FileVersion()) == ConcurrentGreater
}
func (f FileInfo) IsEmpty() bool {

4
lib/protocol/conflict_test.go
View File

@ -14,10 +14,10 @@ func TestWinsConflict(t *testing.T) {
}
for _, tc := range testcases {
if !tc[0].WinsConflict(tc[1]) {
if !WinsConflict(tc[0], tc[1]) {
t.Errorf("%v should win over %v", tc[0], tc[1])
}
if tc[1].WinsConflict(tc[0]) {
if WinsConflict(tc[1], tc[0]) {
t.Errorf("%v should not win over %v", tc[1], tc[0])
}
}