diff --git a/cmd/stindex/idxck.go b/cmd/stindex/idxck.go index 6f5011a08..7d64aaf34 100644 --- a/cmd/stindex/idxck.go +++ b/cmd/stindex/idxck.go @@ -219,10 +219,10 @@ func idxck(ldb backend.Backend) (success bool) { fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name) success = false } - for i, fv := range vl.Versions { - dev, ok := deviceToIDs[string(fv.Device)] + checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) { + dev, ok := deviceToIDs[string(device)] if !ok { - fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device) + fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device) success = false } fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}] @@ -235,14 +235,26 @@ func idxck(ldb backend.Backend) (success bool) { if fi.VersionHash != nil { fiv = versions[string(fi.VersionHash)] } - if !fiv.Equal(fv.Version) { - fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version) + if !fiv.Equal(version) { + fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version) success = false } - if fi.IsInvalid() != fv.Invalid { - fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid()) + if fi.IsInvalid() != invalid { + fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid()) success = false } + if fi.IsDeleted() != deleted { + fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted()) + success = false + } + } + for i, fv := range vl.RawVersions { + for _, device := range fv.Devices { + checkGlobal(i, device, fv.Version, false, fv.Deleted) + } + for _, device := range fv.InvalidDevices { + checkGlobal(i, device, fv.Version, true, fv.Deleted) + } } // If we need this file we should have a need entry for it. False @@ -251,7 +263,9 @@ func idxck(ldb backend.Backend) (success bool) { if needsLocally(vl) { _, ok := needs[gk] if !ok { - dev := deviceToIDs[string(vl.Versions[0].Device)] + fv, _ := vl.GetGlobal() + devB, _ := fv.FirstDevice() + dev := deviceToIDs[string(devB)] fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}] if !fi.IsDeleted() && !fi.IsIgnored() { fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder) @@ -319,15 +333,10 @@ func idxck(ldb backend.Backend) (success bool) { } func needsLocally(vl db.VersionList) bool { - var lv *protocol.Vector - for _, fv := range vl.Versions { - if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) { - lv = &fv.Version - break - } - } - if lv == nil { + fv, ok := vl.Get(protocol.LocalDeviceID[:]) + if !ok { return true // proviosinally, it looks like we need the file } - return !lv.GreaterEqual(vl.Versions[0].Version) + gfv, _ := vl.GetGlobal() // Can't not have a global if we got something above + return !fv.Version.GreaterEqual(gfv.Version) } diff --git a/lib/api/api.go b/lib/api/api.go index dd76e50df..795f40fc9 100644 --- a/lib/api/api.go +++ b/lib/api/api.go @@ -1635,7 +1635,7 @@ func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) { return json.Marshal(m) } -func fileIntfJSONMap(f db.FileIntf) map[string]interface{} { +func fileIntfJSONMap(f protocol.FileIntf) map[string]interface{} { out := map[string]interface{}{ "name": f.FileName(), "type": f.FileType().String(), diff --git a/lib/db/benchmark_test.go b/lib/db/benchmark_test.go index 328df2bc8..e68452369 100644 --- a/lib/db/benchmark_test.go +++ b/lib/db/benchmark_test.go @@ -187,7 +187,7 @@ func BenchmarkNeedHalf(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { count++ return true }) @@ -211,7 +211,7 @@ func BenchmarkNeedHalfRemote(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := fset.Snapshot() - snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool { + snap.WithNeed(remoteDevice0, func(fi protocol.FileIntf) bool { count++ return true }) @@ -232,7 +232,7 @@ func BenchmarkHave(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { count++ return true }) @@ -253,7 +253,7 @@ func BenchmarkGlobal(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithGlobal(func(fi db.FileIntf) bool { + snap.WithGlobal(func(fi protocol.FileIntf) bool { count++ return true }) @@ -274,7 +274,7 @@ func BenchmarkNeedHalfTruncated(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { count++ return true }) @@ -295,7 +295,7 @@ func BenchmarkHaveTruncated(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { count++ return true }) @@ -316,7 +316,7 @@ func BenchmarkGlobalTruncated(b *testing.B) { for i := 0; i < b.N; i++ { count := 0 snap := benchS.Snapshot() - snap.WithGlobalTruncated(func(fi db.FileIntf) bool { + snap.WithGlobalTruncated(func(fi protocol.FileIntf) bool { count++ return true }) diff --git a/lib/db/db_test.go b/lib/db/db_test.go index 6263800ff..b16459edb 100644 --- a/lib/db/db_test.go +++ b/lib/db/db_test.go @@ -183,7 +183,9 @@ func TestUpdate0to3(t *testing.T) { t.Error("File prefixed by '/' was not removed during transition to schema 1") } - key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)) + var key []byte + + key, err = db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid)) if err != nil { t.Fatal(err) } @@ -201,7 +203,7 @@ func TestUpdate0to3(t *testing.T) { t.Fatal(err) } defer trans.Release() - _ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool { + _ = trans.withHaveSequence(folder, 0, func(fi protocol.FileIntf) bool { f := fi.(protocol.FileInfo) l.Infoln(f) if found { @@ -228,12 +230,42 @@ func TestUpdate0to3(t *testing.T) { haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0], haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2], } + trans, err = db.newReadOnlyTransaction() if err != nil { t.Fatal(err) } defer trans.Release() - _ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool { + + key, err = trans.keyer.GenerateNeedFileKey(nil, folder, nil) + if err != nil { + t.Fatal(err) + } + dbi, err := trans.NewPrefixIterator(key) + if err != nil { + t.Fatal(err) + } + defer dbi.Release() + + for dbi.Next() { + name := trans.keyer.NameFromGlobalVersionKey(dbi.Key()) + key, err = trans.keyer.GenerateGlobalVersionKey(key, folder, name) + bs, err := trans.Get(key) + if err != nil { + t.Fatal(err) + } + var vl VersionListDeprecated + if err := vl.Unmarshal(bs); err != nil { + t.Fatal(err) + } + key, err = trans.keyer.GenerateDeviceFileKey(key, folder, vl.Versions[0].Device, name) + fi, ok, err := trans.getFileTrunc(key, false) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("surprise missing global file", string(name), protocol.DeviceIDFromBytes(vl.Versions[0].Device)) + } e, ok := need[fi.FileName()] if !ok { t.Error("Got unexpected needed file:", fi.FileName()) @@ -243,8 +275,11 @@ func TestUpdate0to3(t *testing.T) { if !f.IsEquivalentOptional(e, 0, true, true, 0) { t.Errorf("Wrong needed file, got %v, expected %v", f, e) } - return true - }) + } + if dbi.Error() != nil { + t.Fatal(err) + } + for n := range need { t.Errorf(`Missing needed file "%v"`, n) } @@ -467,7 +502,7 @@ func TestCheckGlobals(t *testing.T) { } // Clean up global entry of the now missing file - if err := db.checkGlobals([]byte(fs.folder), fs.meta); err != nil { + if err := db.checkGlobals([]byte(fs.folder)); err != nil { t.Fatal(err) } @@ -525,7 +560,7 @@ func TestUpdateTo10(t *testing.T) { if err != nil { t.Fatal(err) } - for _, v := range vl.Versions { + for _, v := range vl.RawVersions { if !v.Deleted { t.Error("Unexpected undeleted global version for a") } @@ -535,10 +570,10 @@ func TestUpdateTo10(t *testing.T) { if err != nil { t.Fatal(err) } - if !vl.Versions[0].Deleted { + if !vl.RawVersions[0].Deleted { t.Error("vl.Versions[0] not deleted for b") } - if vl.Versions[1].Deleted { + if vl.RawVersions[1].Deleted { t.Error("vl.Versions[1] deleted for b") } // c @@ -546,10 +581,10 @@ func TestUpdateTo10(t *testing.T) { if err != nil { t.Fatal(err) } - if vl.Versions[0].Deleted { + if vl.RawVersions[0].Deleted { t.Error("vl.Versions[0] deleted for c") } - if !vl.Versions[1].Deleted { + if !vl.RawVersions[1].Deleted { t.Error("vl.Versions[1] not deleted for c") } } diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index 4a35e0779..d3ddc4680 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -426,7 +426,7 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke return t.Commit() } -func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error { +func (db *Lowlevel) checkGlobals(folder []byte) error { t, err := db.newReadWriteTransaction() if err != nil { return err @@ -444,9 +444,10 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error { defer dbi.Release() var dk []byte + ro := t.readOnlyTransaction for dbi.Next() { var vl VersionList - if err := vl.Unmarshal(dbi.Value()); err != nil || len(vl.Versions) == 0 { + if err := vl.Unmarshal(dbi.Value()); err != nil || vl.Empty() { if err := t.Delete(dbi.Key()); err != nil { return err } @@ -459,36 +460,28 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error { // we find those and clear them out. name := db.keyer.NameFromGlobalVersionKey(dbi.Key()) - var newVL VersionList - for i, version := range vl.Versions { - dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, version.Device, name) + newVL := &VersionList{} + var changed, changedHere bool + for _, fv := range vl.RawVersions { + changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.Devices, newVL, ro) if err != nil { return err } - _, err := t.Get(dk) - if backend.IsNotFound(err) { - continue - } - if err != nil { - return err - } - newVL.Versions = append(newVL.Versions, version) + changed = changed || changedHere - if i == 0 { - if fi, ok, err := t.getFileTrunc(dk, true); err != nil { - return err - } else if ok { - meta.addFile(protocol.GlobalDeviceID, fi) - } + changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.InvalidDevices, newVL, ro) + if err != nil { + return err } + changed = changed || changedHere } - if newLen := len(newVL.Versions); newLen == 0 { + if newVL.Empty() { if err := t.Delete(dbi.Key()); err != nil { return err } - } else if newLen != len(vl.Versions) { - if err := t.Put(dbi.Key(), mustMarshal(&newVL)); err != nil { + } else if changed { + if err := t.Put(dbi.Key(), mustMarshal(newVL)); err != nil { return err } } @@ -502,6 +495,30 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error { return t.Commit() } +func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *VersionList, t readOnlyTransaction) (bool, error) { + var changed bool + var err error + for _, device := range devices { + dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, device, name) + if err != nil { + return false, err + } + f, ok, err := t.getFileTrunc(dk, true) + if err != nil { + return false, err + } + if !ok { + changed = true + continue + } + _, _, _, _, _, _, err = vl.update(folder, device, f, t) + if err != nil { + return false, err + } + } + return changed, nil +} + func (db *Lowlevel) getIndexID(device, folder []byte) (protocol.IndexID, error) { key, err := db.keyer.GenerateIndexIDKey(nil, device, folder) if err != nil { @@ -811,7 +828,7 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker { func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) { meta := newMetadataTracker() - if err := db.checkGlobals([]byte(folder), meta); err != nil { + if err := db.checkGlobals([]byte(folder)); err != nil { return nil, err } @@ -831,8 +848,13 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) { return nil, err } + err = t.withGlobal([]byte(folder), nil, true, func(f protocol.FileIntf) bool { + meta.addFile(protocol.GlobalDeviceID, f) + return true + }) + meta.emptyNeeded(protocol.LocalDeviceID) - err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f FileIntf) bool { + err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f protocol.FileIntf) bool { meta.addNeeded(protocol.LocalDeviceID, f) return true }) @@ -841,7 +863,7 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) { } for _, device := range meta.devices() { meta.emptyNeeded(device) - err = t.withNeed([]byte(folder), device[:], true, func(f FileIntf) bool { + err = t.withNeed([]byte(folder), device[:], true, func(f protocol.FileIntf) bool { meta.addNeeded(device, f) return true }) @@ -878,7 +900,7 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool { panic(err) } ok := true - if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi FileIntf) bool { + if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool { ok = false // we got something, which we should not have return false }); err != nil && !backend.IsClosed(err) { @@ -1004,6 +1026,6 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack // unchanged checks if two files are the same and thus don't need to be updated. // Local flags or the invalid bit might change without the version // being bumped. -func unchanged(nf, ef FileIntf) bool { +func unchanged(nf, ef protocol.FileIntf) bool { return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags() } diff --git a/lib/db/meta.go b/lib/db/meta.go index b8a462ae2..baa376aed 100644 --- a/lib/db/meta.go +++ b/lib/db/meta.go @@ -147,7 +147,7 @@ func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts { // addFile adds a file to the counts, adjusting the sequence number as // appropriate -func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) { +func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileIntf) { m.mut.Lock() defer m.mut.Unlock() @@ -186,7 +186,7 @@ func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) { } // addNeeded adds a file to the needed counts -func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f FileIntf) { +func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileIntf) { m.mut.Lock() defer m.mut.Unlock() @@ -201,7 +201,7 @@ func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 { return m.countsPtr(dev, 0).Sequence } -func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) { +func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileIntf) { if dev == protocol.GlobalDeviceID { return } @@ -210,7 +210,7 @@ func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) { } } -func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) { +func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) { cp := m.countsPtr(dev, flag) switch { @@ -227,7 +227,7 @@ func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f Fi } // removeFile removes a file from the counts -func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { +func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileIntf) { if f.IsInvalid() && f.FileLocalFlags() == 0 { // This is a remote invalid file; it does not count. return @@ -250,7 +250,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) { } // removeNeeded removes a file from the needed counts -func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) { +func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileIntf) { m.mut.Lock() defer m.mut.Unlock() @@ -259,7 +259,7 @@ func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) { m.removeFileLocked(dev, needFlag, f) } -func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) { +func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) { cp := m.countsPtr(dev, flag) switch { diff --git a/lib/db/schemaupdater.go b/lib/db/schemaupdater.go index 16ac28dcc..99e3f26a3 100644 --- a/lib/db/schemaupdater.go +++ b/lib/db/schemaupdater.go @@ -7,7 +7,10 @@ package db import ( + "bytes" + "errors" "fmt" + "sort" "strings" "github.com/syncthing/syncthing/lib/db/backend" @@ -23,12 +26,14 @@ import ( // 7: v0.14.53 // 8-9: v1.4.0 // 10-11: v1.6.0 -// 12: v1.7.0 +// 12-13: v1.7.0 const ( - dbVersion = 12 + dbVersion = 13 dbMinSyncthingVersion = "v1.7.0" ) +var errFolderMissing = errors.New("folder present in global list but missing in keyer index") + type databaseDowngradeError struct { minSyncthingVersion string } @@ -89,14 +94,14 @@ func (db *schemaUpdater) updateSchema() error { {9, db.updateSchemaTo9}, {10, db.updateSchemaTo10}, {11, db.updateSchemaTo11}, - {12, db.updateSchemaTo12}, + {13, db.updateSchemaTo13}, } for _, m := range migrations { if prevVersion < m.schemaVersion { l.Infof("Migrating database to schema version %d...", m.schemaVersion) if err := m.migration(int(prevVersion)); err != nil { - return err + return fmt.Errorf("failed migrating to version %v: %w", m.schemaVersion, err) } } } @@ -128,8 +133,8 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { symlinkConv := 0 changedFolders := make(map[string]struct{}) ignAdded := 0 - meta := newMetadataTracker() // dummy metadata tracker - var gk, buf []byte + var gk []byte + ro := t.readOnlyTransaction for dbi.Next() { folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key()) @@ -155,17 +160,27 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { if _, ok := changedFolders[string(folder)]; !ok { changedFolders[string(folder)] = struct{}{} } + if err := t.Delete(dbi.Key()); err != nil { + return err + } gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name) if err != nil { return err } - // Purposely pass nil file name to remove from global list, - // but don't touch meta and needs - buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil) - if err != nil && err != errEntryFromGlobalMissing { + fl, err := getGlobalVersionsByKeyBefore11(gk, ro) + if backend.IsNotFound(err) { + // Shouldn't happen, but not critical. + continue + } else if err != nil { return err } - if err := t.Delete(dbi.Key()); err != nil { + _, _ = fl.pop(device) + if len(fl.Versions) == 0 { + err = t.Delete(gk) + } else { + err = t.Put(gk, mustMarshal(&fl)) + } + if err != nil { return err } continue @@ -199,13 +214,41 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { if err != nil { return err } - if buf, ok, err = t.updateGlobal(gk, buf, folder, device, f, meta); err != nil { + + fl, err := getGlobalVersionsByKeyBefore11(gk, ro) + if err != nil && !backend.IsNotFound(err) { return err - } else if ok { - if _, ok = changedFolders[string(folder)]; !ok { - changedFolders[string(folder)] = struct{}{} + } + i := 0 + i = sort.Search(len(fl.Versions), func(j int) bool { + return fl.Versions[j].Invalid + }) + for ; i < len(fl.Versions); i++ { + ordering := fl.Versions[i].Version.Compare(f.Version) + shouldInsert := ordering == protocol.Equal + if !shouldInsert { + shouldInsert, err = shouldInsertBefore(ordering, folder, fl.Versions[i].Device, true, f, ro) + if err != nil { + return err + } } - ignAdded++ + if shouldInsert { + nv := FileVersionDeprecated{ + Device: device, + Version: f.Version, + Invalid: true, + } + fl.insertAt(i, nv) + if err := t.Put(gk, mustMarshal(&fl)); err != nil { + return err + } + if _, ok := changedFolders[string(folder)]; !ok { + changedFolders[string(folder)] = struct{}{} + } + ignAdded++ + break + } + } } if err := t.Checkpoint(); err != nil { @@ -217,11 +260,6 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error { return err } - for folder := range changedFolders { - if err := db.dropFolderMeta([]byte(folder)); err != nil { - return err - } - } return t.Commit() } @@ -239,7 +277,7 @@ func (db *schemaUpdater) updateSchema1to2(_ int) error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool { + err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f protocol.FileIntf) bool { sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo()) if putErr != nil { return false @@ -274,7 +312,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := t.withGlobal(folder, nil, true, func(f FileIntf) bool { + err := withGlobalBefore11(folder, true, func(f protocol.FileIntf) bool { name := []byte(f.FileName()) dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name) if putErr != nil { @@ -289,12 +327,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error { if ok { v = haveFile.FileVersion() } - fv := FileVersion{ + fv := FileVersionDeprecated{ Version: f.FileVersion(), Invalid: f.IsInvalid(), Deleted: f.IsDeleted(), } - if !need(fv, ok, v) { + if !needDeprecated(fv, ok, v) { return true } nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName())) @@ -303,7 +341,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error { } putErr = t.Put(nk, nil) return putErr == nil - }) + }, t.readOnlyTransaction) if putErr != nil { return putErr } @@ -359,7 +397,7 @@ func (db *schemaUpdater) updateSchema5to6(_ int) error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var iterErr error - err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool { + err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f protocol.FileIntf) bool { if !f.IsInvalid() { return true } @@ -404,7 +442,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var delErr error - err := t.withNeedLocal(folder, false, func(f FileIntf) bool { + err := withNeedLocalBefore11(folder, false, func(f protocol.FileIntf) bool { name := []byte(f.FileName()) gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name) if delErr != nil { @@ -421,20 +459,20 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { delErr = t.Delete(key) return delErr == nil } - var fl VersionList + var fl VersionListDeprecated err = fl.Unmarshal(svl) if err != nil { // This can't happen, but it's ignored everywhere else too, // so lets not act on it. return true } - globalFV := FileVersion{ + globalFV := FileVersionDeprecated{ Version: f.FileVersion(), Invalid: f.IsInvalid(), Deleted: f.IsDeleted(), } - if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) { + if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !needDeprecated(globalFV, haveLocalFV, localFV.Version) { key, err := t.keyer.GenerateNeedFileKey(nk, folder, name) if err != nil { delErr = err @@ -443,7 +481,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error { delErr = t.Delete(key) } return delErr == nil - }) + }, t.readOnlyTransaction) if delErr != nil { return delErr } @@ -480,6 +518,7 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error { if err != nil { return err } + defer it.Release() for it.Next() { intf, err := t.unmarshalTrunc(it.Value(), false) if backend.IsNotFound(err) { @@ -510,6 +549,8 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error { } func (db *schemaUpdater) updateSchemaTo10(_ int) error { + // Rewrites global lists to include a Deleted flag. + t, err := db.newReadWriteTransaction() if err != nil { return err @@ -533,7 +574,7 @@ func (db *schemaUpdater) updateSchemaTo10(_ int) error { defer dbi.Release() for dbi.Next() { - var vl VersionList + var vl VersionListDeprecated if err := vl.Unmarshal(dbi.Value()); err != nil { return err } @@ -592,7 +633,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error { for _, folderStr := range db.ListFolders() { folder := []byte(folderStr) var putErr error - err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi FileIntf) bool { + err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi protocol.FileIntf) bool { f := fi.(FileInfoTruncated) if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() || f.IsInvalid() || f.BlocksHash == nil { return true @@ -620,7 +661,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error { return t.Commit() } -func (db *schemaUpdater) updateSchemaTo12(_ int) error { +func (db *schemaUpdater) updateSchemaTo13(prev int) error { // Loads and rewrites all files, to deduplicate version vectors. t, err := db.newReadWriteTransaction() @@ -629,9 +670,280 @@ func (db *schemaUpdater) updateSchemaTo12(_ int) error { } defer t.close() - if err := db.rewriteFiles(t); err != nil { + if prev < 12 { + if err := db.rewriteFiles(t); err != nil { + return err + } + } + + if err := db.rewriteGlobals(t); err != nil { return err } return t.Commit() } + +func (db *schemaUpdater) rewriteGlobals(t readWriteTransaction) error { + it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal}) + if err != nil { + return err + } + defer it.Release() + for it.Next() { + var vl VersionListDeprecated + if err := vl.Unmarshal(it.Value()); err != nil { + // If we crashed during an earlier migration, some version + // lists might already be in the new format: Skip those. + var nvl VersionList + if nerr := nvl.Unmarshal(it.Value()); nerr == nil { + continue + } + return err + } + if len(vl.Versions) == 0 { + if err := t.Delete(it.Key()); err != nil { + return err + } + } + + newVl, err := convertVersionList(vl) + if err != nil { + return err + } + if err := t.Put(it.Key(), mustMarshal(&newVl)); err != nil { + return err + } + if err := t.Checkpoint(); err != nil { + return err + } + } + it.Release() + return it.Error() +} + +func convertVersionList(vl VersionListDeprecated) (VersionList, error) { + var newVl VersionList + var newPos, oldPos int + var lastVersion protocol.Vector + + for _, fv := range vl.Versions { + if fv.Invalid { + break + } + oldPos++ + if lastVersion.Equal(fv.Version) { + newVl.RawVersions[newPos].Devices = append(newVl.RawVersions[newPos].Devices, fv.Device) + continue + } + newPos = len(newVl.RawVersions) + newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, false, fv.Deleted)) + lastVersion = fv.Version + } + + if oldPos == len(vl.Versions) { + return newVl, nil + } + + if len(newVl.RawVersions) == 0 { + fv := vl.Versions[oldPos] + newVl.RawVersions = []FileVersion{newFileVersion(fv.Device, fv.Version, true, fv.Deleted)} + } + newPos = 0 +outer: + for _, fv := range vl.Versions[oldPos:] { + for _, nfv := range newVl.RawVersions[newPos:] { + switch nfv.Version.Compare(fv.Version) { + case protocol.Equal: + newVl.RawVersions[newPos].InvalidDevices = append(newVl.RawVersions[newPos].InvalidDevices, fv.Device) + lastVersion = fv.Version + continue outer + case protocol.Lesser: + newVl.insertAt(newPos, newFileVersion(fv.Device, fv.Version, true, fv.Deleted)) + lastVersion = fv.Version + continue outer + case protocol.ConcurrentLesser, protocol.ConcurrentGreater: + // The version is invalid, i.e. it looses anyway, + // no need to check/get the conflicting file. + } + newPos++ + } + // Couldn't insert into any existing versions + newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, true, fv.Deleted)) + lastVersion = fv.Version + newPos++ + } + + return newVl, nil +} + +func getGlobalVersionsByKeyBefore11(key []byte, t readOnlyTransaction) (VersionListDeprecated, error) { + bs, err := t.Get(key) + if err != nil { + return VersionListDeprecated{}, err + } + + var vl VersionListDeprecated + if err := vl.Unmarshal(bs); err != nil { + return VersionListDeprecated{}, err + } + + return vl, nil +} + +func withGlobalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error { + key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key) + if err != nil { + return err + } + defer dbi.Release() + + var dk []byte + for dbi.Next() { + name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) + + var vl VersionListDeprecated + if err := vl.Unmarshal(dbi.Value()); err != nil { + return err + } + + dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name) + if err != nil { + return err + } + + f, ok, err := t.getFileTrunc(dk, truncate) + if err != nil { + return err + } + if !ok { + continue + } + + if !fn(f) { + return nil + } + } + if err != nil { + return err + } + return dbi.Error() +} + +func withNeedLocalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error { + key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil) + if err != nil { + return err + } + dbi, err := t.NewPrefixIterator(key.WithoutName()) + if err != nil { + return err + } + defer dbi.Release() + + var keyBuf []byte + var f protocol.FileIntf + var ok bool + for dbi.Next() { + keyBuf, f, ok, err = getGlobalBefore11(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate, t) + if err != nil { + return err + } + if !ok { + continue + } + if !fn(f) { + return nil + } + } + return dbi.Error() +} + +func getGlobalBefore11(keyBuf, folder, file []byte, truncate bool, t readOnlyTransaction) ([]byte, protocol.FileIntf, bool, error) { + keyBuf, err := t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file) + if err != nil { + return nil, nil, false, err + } + bs, err := t.Get(keyBuf) + if backend.IsNotFound(err) { + return keyBuf, nil, false, nil + } else if err != nil { + return nil, nil, false, err + } + var vl VersionListDeprecated + if err := vl.Unmarshal(bs); err != nil { + return nil, nil, false, err + } + if len(vl.Versions) == 0 { + return nil, nil, false, nil + } + keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file) + if err != nil { + return nil, nil, false, err + } + fi, ok, err := t.getFileTrunc(keyBuf, truncate) + if err != nil || !ok { + return keyBuf, nil, false, err + } + return keyBuf, fi, true, nil +} + +func (vl *VersionListDeprecated) String() string { + var b bytes.Buffer + var id protocol.DeviceID + b.WriteString("{") + for i, v := range vl.Versions { + if i > 0 { + b.WriteString(", ") + } + copy(id[:], v.Device) + fmt.Fprintf(&b, "{%v, %v}", v.Version, id) + } + b.WriteString("}") + return b.String() +} + +func (vl *VersionListDeprecated) pop(device []byte) (FileVersionDeprecated, int) { + for i, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...) + return v, i + } + } + return FileVersionDeprecated{}, -1 +} + +func (vl *VersionListDeprecated) Get(device []byte) (FileVersionDeprecated, bool) { + for _, v := range vl.Versions { + if bytes.Equal(v.Device, device) { + return v, true + } + } + + return FileVersionDeprecated{}, false +} + +func (vl *VersionListDeprecated) insertAt(i int, v FileVersionDeprecated) { + vl.Versions = append(vl.Versions, FileVersionDeprecated{}) + copy(vl.Versions[i+1:], vl.Versions[i:]) + vl.Versions[i] = v +} + +func needDeprecated(global FileVersionDeprecated, haveLocal bool, localVersion protocol.Vector) bool { + // We never need an invalid file. + if global.Invalid { + return false + } + // We don't need a deleted file if we don't have it. + if global.Deleted && !haveLocal { + return false + } + // We don't need the global file if we already have the same version. + if haveLocal && localVersion.GreaterEqual(global.Version) { + return false + } + return true +} diff --git a/lib/db/set.go b/lib/db/set.go index e29c64f59..543f14606 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -13,8 +13,6 @@ package db import ( - "time" - "github.com/syncthing/syncthing/lib/db/backend" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" @@ -31,35 +29,10 @@ type FileSet struct { updateMutex sync.Mutex // protects database updates and the corresponding metadata changes } -// FileIntf is the set of methods implemented by both protocol.FileInfo and -// FileInfoTruncated. -type FileIntf interface { - FileSize() int64 - FileName() string - FileLocalFlags() uint32 - IsDeleted() bool - IsInvalid() bool - IsIgnored() bool - IsUnsupported() bool - MustRescan() bool - IsReceiveOnlyChanged() bool - IsDirectory() bool - IsSymlink() bool - ShouldConflict() bool - HasPermissionBits() bool - SequenceNo() int64 - BlockSize() int - FileVersion() protocol.Vector - FileType() protocol.FileInfoType - FilePermissions() uint32 - FileModifiedBy() protocol.ShortID - ModTime() time.Time -} - // The Iterator is called with either a protocol.FileInfo or a // FileInfoTruncated (depending on the method) and returns true to // continue iteration, false to stop. -type Iterator func(f FileIntf) bool +type Iterator func(f protocol.FileIntf) bool func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet { return &FileSet{ @@ -335,7 +308,7 @@ func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated { skip := (page - 1) * perpage get := perpage - s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool { + s.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { if !f.IsReceiveOnlyChanged() { return true } @@ -359,7 +332,7 @@ func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage files := make([]FileInfoTruncated, 0, perpage) skip := (page - 1) * perpage get := perpage - s.WithNeedTruncated(device, func(f FileIntf) bool { + s.WithNeedTruncated(device, func(f protocol.FileIntf) bool { if skip > 0 { skip-- return true @@ -497,7 +470,7 @@ func normalizeFilenamesAndDropDuplicates(fs []protocol.FileInfo) []protocol.File } func nativeFileIterator(fn Iterator) Iterator { - return func(fi FileIntf) bool { + return func(fi protocol.FileIntf) bool { switch f := fi.(type) { case protocol.FileInfo: f.Name = osutil.NativeFilename(f.Name) diff --git a/lib/db/set_test.go b/lib/db/set_test.go index bd5f2346c..d97a66bcc 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -48,7 +48,7 @@ func globalList(s *db.FileSet) []protocol.FileInfo { var fs []protocol.FileInfo snap := s.Snapshot() defer snap.Release() - snap.WithGlobal(func(fi db.FileIntf) bool { + snap.WithGlobal(func(fi protocol.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -59,7 +59,7 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated { var fs []db.FileInfoTruncated snap := s.Snapshot() defer snap.Release() - snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { + snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool { f := fi.(db.FileInfoTruncated) fs = append(fs, f) return true @@ -71,7 +71,7 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo snap := s.Snapshot() defer snap.Release() - snap.WithHave(n, func(fi db.FileIntf) bool { + snap.WithHave(n, func(fi protocol.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -83,7 +83,7 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi var fs []db.FileInfoTruncated snap := s.Snapshot() defer snap.Release() - snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool { + snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileIntf) bool { f := fi.(db.FileInfoTruncated) fs = append(fs, f) return true @@ -95,7 +95,7 @@ func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo { var fs []protocol.FileInfo snap := s.Snapshot() defer snap.Release() - snap.WithNeed(n, func(fi db.FileIntf) bool { + snap.WithNeed(n, func(fi protocol.FileIntf) bool { f := fi.(protocol.FileInfo) fs = append(fs, f) return true @@ -998,7 +998,7 @@ func TestWithHaveSequence(t *testing.T) { i := 2 snap := s.Snapshot() defer snap.Release() - snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool { + snap.WithHaveSequence(int64(i), func(fi protocol.FileIntf) bool { if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) { t.Fatalf("Got %v\nExpected %v", f, localHave[i-1]) } @@ -1049,7 +1049,7 @@ loop: default: } snap := s.Snapshot() - snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool { + snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileIntf) bool { if fi.SequenceNo() < prevSeq+1 { t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo()) } @@ -1527,8 +1527,8 @@ func TestSequenceIndex(t *testing.T) { // Start a routine to walk the sequence index and inspect the result. - seen := make(map[string]db.FileIntf) - latest := make([]db.FileIntf, 0, len(local)) + seen := make(map[string]protocol.FileIntf) + latest := make([]protocol.FileIntf, 0, len(local)) var seq int64 t0 := time.Now() @@ -1539,7 +1539,7 @@ func TestSequenceIndex(t *testing.T) { // update has happened since our last iteration. latest = latest[:0] snap := s.Snapshot() - snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool { + snap.WithHaveSequence(seq+1, func(f protocol.FileIntf) bool { seen[f.FileName()] = f latest = append(latest, f) seq = f.SequenceNo() @@ -1644,7 +1644,7 @@ func TestUpdateWithOneFileTwice(t *testing.T) { snap := s.Snapshot() defer snap.Release() count := 0 - snap.WithHaveSequence(0, func(f db.FileIntf) bool { + snap.WithHaveSequence(0, func(f protocol.FileIntf) bool { count++ return true }) diff --git a/lib/db/structs.go b/lib/db/structs.go index 8c8c60c18..a744b1e64 100644 --- a/lib/db/structs.go +++ b/lib/db/structs.go @@ -12,7 +12,6 @@ package db import ( "bytes" "fmt" - "sort" "time" "github.com/syncthing/syncthing/lib/protocol" @@ -196,98 +195,317 @@ func (vl VersionList) String() string { var b bytes.Buffer var id protocol.DeviceID b.WriteString("{") - for i, v := range vl.Versions { + for i, v := range vl.RawVersions { if i > 0 { b.WriteString(", ") } - copy(id[:], v.Device) - fmt.Fprintf(&b, "{%v, %v}", v.Version, id) + fmt.Fprintf(&b, "{%v, {", v.Version) + for j, dev := range v.Devices { + if j > 0 { + b.WriteString(", ") + } + copy(id[:], dev) + fmt.Fprint(&b, id.Short()) + } + b.WriteString("}, {") + for j, dev := range v.InvalidDevices { + if j > 0 { + b.WriteString(", ") + } + copy(id[:], dev) + fmt.Fprint(&b, id.Short()) + } + fmt.Fprint(&b, "}}") } b.WriteString("}") return b.String() } // update brings the VersionList up to date with file. It returns the updated -// VersionList, a potentially removed old FileVersion and its index, as well as -// the index where the new FileVersion was inserted. -func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int, err error) { - vl, removedFV, removedAt = vl.pop(device) - - nv := FileVersion{ - Device: device, - Version: file.Version, - Invalid: file.IsInvalid(), - Deleted: file.IsDeleted(), - } - i := 0 - if nv.Invalid { - i = sort.Search(len(vl.Versions), func(j int) bool { - return vl.Versions[j].Invalid - }) - } - for ; i < len(vl.Versions); i++ { - switch vl.Versions[i].Version.Compare(file.Version) { - case protocol.Equal: - fallthrough - - case protocol.Lesser: - // The version at this point in the list is equal to or lesser - // ("older") than us. We insert ourselves in front of it. - vl = vl.insertAt(i, nv) - return vl, removedFV, removedAt, i, nil - - case protocol.ConcurrentLesser, protocol.ConcurrentGreater: - // The version at this point is in conflict with us. We must pull - // the actual file metadata to determine who wins. If we win, we - // insert ourselves in front of the loser here. (The "Lesser" and - // "Greater" in the condition above is just based on the device - // IDs in the version vector, which is not the only thing we use - // to determine the winner.) - // - // A surprise missing file entry here is counted as a win for us. - if of, ok, err := t.getFile(folder, vl.Versions[i].Device, []byte(file.Name)); err != nil { - return vl, removedFV, removedAt, i, err - } else if !ok || file.WinsConflict(of) { - vl = vl.insertAt(i, nv) - return vl, removedFV, removedAt, i, nil - } - } +// VersionList, a device that has the global/newest version, a device that previously +// had the global/newest version, a boolean indicating if the global version has +// changed and if any error occurred (only possible in db interaction). +func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) { + if len(vl.RawVersions) == 0 { + nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()) + vl.RawVersions = append(vl.RawVersions, nv) + return nv, FileVersion{}, FileVersion{}, false, false, true, nil } - // We didn't find a position for an insert above, so append to the end. - vl.Versions = append(vl.Versions, nv) + // Get the current global (before updating) + oldFV, haveOldGlobal := vl.GetGlobal() - return vl, removedFV, removedAt, len(vl.Versions) - 1, nil + // Remove ourselves first + removedFV, haveRemoved, _, err := vl.pop(folder, device, []byte(file.FileName()), t) + if err == nil { + // Find position and insert the file + err = vl.insert(folder, device, file, t) + } + if err != nil { + return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err + } + + newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty + + if !haveOldGlobal { + return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil + } + + globalChanged := true + if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) { + globalChanged = false + } + + return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil } -func (vl VersionList) insertAt(i int, v FileVersion) VersionList { - vl.Versions = append(vl.Versions, FileVersion{}) - copy(vl.Versions[i+1:], vl.Versions[i:]) - vl.Versions[i] = v - return vl +func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error { + var added bool + var err error + i := 0 + for ; i < len(vl.RawVersions); i++ { + // Insert our new version + added, err = vl.checkInsertAt(i, folder, device, file, t) + if err != nil { + return err + } + if added { + break + } + } + if i == len(vl.RawVersions) { + // Append to the end + vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())) + } + return nil +} + +func (vl *VersionList) insertAt(i int, v FileVersion) { + vl.RawVersions = append(vl.RawVersions, FileVersion{}) + copy(vl.RawVersions[i+1:], vl.RawVersions[i:]) + vl.RawVersions[i] = v } // pop returns the VersionList without the entry for the given device, as well -// as the removed FileVersion and the position, where that FileVersion was. -// If there is no FileVersion for the given device, the position is -1. -func (vl VersionList) pop(device []byte) (VersionList, FileVersion, int) { - for i, v := range vl.Versions { - if bytes.Equal(v.Device, device) { - vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...) - return vl, v, i - } +// as the removed FileVersion, whether it was found/removed at all and whether +// the global changed in the process. +func (vl *VersionList) pop(folder, device, name []byte, t readOnlyTransaction) (FileVersion, bool, bool, error) { + invDevice, i, j, ok := vl.findDevice(device) + if !ok { + return FileVersion{}, false, false, nil } - return vl, FileVersion{}, -1 + globalPos := vl.findGlobal() + + if vl.RawVersions[i].deviceCount() == 1 { + fv := vl.RawVersions[i] + vl.popVersionAt(i) + return fv, true, globalPos == i, nil + } + + if invDevice { + vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j) + } else { + vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j) + } + // If the last valid device of the previous global was removed above, + // the next entry is now the global entry (unless all entries are invalid). + if len(vl.RawVersions[i].Devices) == 0 && globalPos == i { + return vl.RawVersions[i], true, globalPos == vl.findGlobal(), nil + } + return vl.RawVersions[i], true, false, nil } -func (vl VersionList) Get(device []byte) (FileVersion, bool) { - for _, v := range vl.Versions { - if bytes.Equal(v.Device, device) { - return v, true +// Get returns a FileVersion that contains the given device and whether it has +// been found at all. +func (vl *VersionList) Get(device []byte) (FileVersion, bool) { + _, i, _, ok := vl.findDevice(device) + if !ok { + return FileVersion{}, false + } + return vl.RawVersions[i], true +} + +// GetGlobal returns the current global FileVersion. The returned FileVersion +// may be invalid, if all FileVersions are invalid. Returns false only if +// VersionList is empty. +func (vl *VersionList) GetGlobal() (FileVersion, bool) { + i := vl.findGlobal() + if i == -1 { + return FileVersion{}, false + } + return vl.RawVersions[i], true +} + +func (vl *VersionList) Empty() bool { + return len(vl.RawVersions) == 0 +} + +// findGlobal returns the first version that isn't invalid, or if all versions are +// invalid just the first version (i.e. 0) or -1, if there's no versions at all. +func (vl *VersionList) findGlobal() int { + for i, fv := range vl.RawVersions { + if !fv.IsInvalid() { + return i } } + if len(vl.RawVersions) == 0 { + return -1 + } + return 0 +} - return FileVersion{}, false +// findDevices returns whether the device is in InvalidVersions or Versions and +// in InvalidDevices or Devices (true for invalid), the positions in the version +// and device slices and whether it has been found at all. +func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) { + for i, v := range vl.RawVersions { + if j := deviceIndex(v.Devices, device); j != -1 { + return false, i, j, true + } + if j := deviceIndex(v.InvalidDevices, device); j != -1 { + return true, i, j, true + } + } + return false, -1, -1, false +} + +func (vl *VersionList) popVersion(version protocol.Vector) (FileVersion, bool) { + i := vl.versionIndex(version) + if i == -1 { + return FileVersion{}, false + } + fv := vl.RawVersions[i] + vl.popVersionAt(i) + return fv, true +} + +func (vl *VersionList) versionIndex(version protocol.Vector) int { + for i, v := range vl.RawVersions { + if version.Equal(v.Version) { + return i + } + } + return -1 +} + +func (vl *VersionList) popVersionAt(i int) { + vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...) +} + +// checkInsertAt determines if the given device and associated file should be +// inserted into the FileVersion at position i or into a new FileVersion at +// position i. +func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) { + ordering := vl.RawVersions[i].Version.Compare(file.FileVersion()) + if ordering == protocol.Equal { + if !file.IsInvalid() { + vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device) + } else { + vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device) + } + return true, nil + } + existingDevice, _ := vl.RawVersions[i].FirstDevice() + insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t) + if err != nil { + return false, err + } + if insert { + vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())) + return true, nil + } + return false, nil +} + +// shouldInsertBefore determines whether the file comes before an existing +// entry, given the version ordering (existing compared to new one), existing +// device and if the existing version is invalid. +func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) { + switch ordering { + case protocol.Lesser: + // The version at this point in the list is lesser + // ("older") than us. We insert ourselves in front of it. + return true, nil + + case protocol.ConcurrentLesser, protocol.ConcurrentGreater: + // The version in conflict with us. + // Check if we can shortcut due to one being invalid. + if existingInvalid != file.IsInvalid() { + return existingInvalid, nil + } + // We must pull the actual file metadata to determine who wins. + // If we win, we insert ourselves in front of the loser here. + // (The "Lesser" and "Greater" in the condition above is just + // based on the device IDs in the version vector, which is not + // the only thing we use to determine the winner.) + of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName())) + if err != nil { + return false, err + } + // A surprise missing file entry here is counted as a win for us. + if !ok { + return true, nil + } + if err != nil { + return false, err + } + if protocol.WinsConflict(file, of) { + return true, nil + } + } + return false, nil +} + +func deviceIndex(devices [][]byte, device []byte) int { + for i, dev := range devices { + if bytes.Equal(device, dev) { + return i + } + } + return -1 +} + +func popDeviceAt(devices [][]byte, i int) [][]byte { + return append(devices[:i], devices[i+1:]...) +} + +func popDevice(devices [][]byte, device []byte) ([][]byte, bool) { + i := deviceIndex(devices, device) + if i == -1 { + return devices, false + } + return popDeviceAt(devices, i), true +} + +func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion { + fv := FileVersion{ + Version: version, + Deleted: deleted, + } + if invalid { + fv.InvalidDevices = [][]byte{device} + } else { + fv.Devices = [][]byte{device} + } + return fv +} + +func (fv FileVersion) FirstDevice() ([]byte, bool) { + if len(fv.Devices) != 0 { + return fv.Devices[0], true + } + if len(fv.InvalidDevices) != 0 { + return fv.InvalidDevices[0], true + } + return nil, false +} + +func (fv FileVersion) IsInvalid() bool { + return len(fv.Devices) == 0 +} + +func (fv FileVersion) deviceCount() int { + return len(fv.Devices) + len(fv.InvalidDevices) } type fileList []protocol.FileInfo diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go index d343c80e6..0747c2a99 100644 --- a/lib/db/structs.pb.go +++ b/lib/db/structs.pb.go @@ -26,10 +26,10 @@ var _ = math.Inf const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type FileVersion struct { - Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` - Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` - Invalid bool `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"` - Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` + Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` + Deleted bool `protobuf:"varint,2,opt,name=deleted,proto3" json:"deleted,omitempty"` + Devices [][]byte `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"` + InvalidDevices [][]byte `protobuf:"bytes,4,rep,name=invalid_devices,json=invalidDevices,proto3" json:"invalid_devices,omitempty"` } func (m *FileVersion) Reset() { *m = FileVersion{} } @@ -66,7 +66,7 @@ func (m *FileVersion) XXX_DiscardUnknown() { var xxx_messageInfo_FileVersion proto.InternalMessageInfo type VersionList struct { - Versions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"` + RawVersions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"` } func (m *VersionList) Reset() { *m = VersionList{} } @@ -318,6 +318,82 @@ func (m *CountsSet) XXX_DiscardUnknown() { var xxx_messageInfo_CountsSet proto.InternalMessageInfo +type FileVersionDeprecated struct { + Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"` + Device []byte `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` + Invalid bool `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"` + Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` +} + +func (m *FileVersionDeprecated) Reset() { *m = FileVersionDeprecated{} } +func (m *FileVersionDeprecated) String() string { return proto.CompactTextString(m) } +func (*FileVersionDeprecated) ProtoMessage() {} +func (*FileVersionDeprecated) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{7} +} +func (m *FileVersionDeprecated) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FileVersionDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FileVersionDeprecated.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FileVersionDeprecated) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileVersionDeprecated.Merge(m, src) +} +func (m *FileVersionDeprecated) XXX_Size() int { + return m.ProtoSize() +} +func (m *FileVersionDeprecated) XXX_DiscardUnknown() { + xxx_messageInfo_FileVersionDeprecated.DiscardUnknown(m) +} + +var xxx_messageInfo_FileVersionDeprecated proto.InternalMessageInfo + +type VersionListDeprecated struct { + Versions []FileVersionDeprecated `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"` +} + +func (m *VersionListDeprecated) Reset() { *m = VersionListDeprecated{} } +func (*VersionListDeprecated) ProtoMessage() {} +func (*VersionListDeprecated) Descriptor() ([]byte, []int) { + return fileDescriptor_e774e8f5f348d14d, []int{8} +} +func (m *VersionListDeprecated) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VersionListDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VersionListDeprecated.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VersionListDeprecated) XXX_Merge(src proto.Message) { + xxx_messageInfo_VersionListDeprecated.Merge(m, src) +} +func (m *VersionListDeprecated) XXX_Size() int { + return m.ProtoSize() +} +func (m *VersionListDeprecated) XXX_DiscardUnknown() { + xxx_messageInfo_VersionListDeprecated.DiscardUnknown(m) +} + +var xxx_messageInfo_VersionListDeprecated proto.InternalMessageInfo + func init() { proto.RegisterType((*FileVersion)(nil), "db.FileVersion") proto.RegisterType((*VersionList)(nil), "db.VersionList") @@ -326,61 +402,68 @@ func init() { proto.RegisterType((*IndirectionHashesOnly)(nil), "db.IndirectionHashesOnly") proto.RegisterType((*Counts)(nil), "db.Counts") proto.RegisterType((*CountsSet)(nil), "db.CountsSet") + proto.RegisterType((*FileVersionDeprecated)(nil), "db.FileVersionDeprecated") + proto.RegisterType((*VersionListDeprecated)(nil), "db.VersionListDeprecated") } func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) } var fileDescriptor_e774e8f5f348d14d = []byte{ - // 774 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x8f, 0xe3, 0x44, - 0x10, 0x8d, 0x37, 0x71, 0x3e, 0xca, 0x49, 0xd8, 0x6d, 0x96, 0x91, 0x15, 0x09, 0xc7, 0x0a, 0x5a, - 0xc9, 0xe2, 0x90, 0xc0, 0xec, 0x0d, 0x24, 0x0e, 0x61, 0x35, 0x22, 0x12, 0x62, 0x51, 0x67, 0xb5, - 0xa7, 0x95, 0x22, 0x7f, 0x74, 0x92, 0xd6, 0x38, 0xee, 0xe0, 0xee, 0xcc, 0xc8, 0xf3, 0x17, 0xb8, - 0x70, 0xe4, 0x38, 0x17, 0xfe, 0xcb, 0x1c, 0xe7, 0x88, 0x38, 0x44, 0x90, 0x70, 0x80, 0x7f, 0x81, - 0xba, 0xdb, 0x76, 0x3c, 0x73, 0x61, 0x6e, 0x55, 0xaf, 0x2a, 0xa9, 0xaa, 0xf7, 0x9e, 0x1b, 0x7a, - 0x5c, 0xa4, 0xbb, 0x50, 0xf0, 0xf1, 0x36, 0x65, 0x82, 0xa1, 0x67, 0x51, 0x30, 0xf8, 0x2c, 0x25, - 0x5b, 0xc6, 0x27, 0x0a, 0x08, 0x76, 0xcb, 0xc9, 0x8a, 0xad, 0x98, 0x4a, 0x54, 0xa4, 0x1b, 0x07, - 0x67, 0x31, 0x0d, 0x74, 0x4b, 0xc8, 0xe2, 0x49, 0x40, 0xb6, 0x1a, 0x1f, 0xfd, 0x6c, 0x80, 0x75, - 0x41, 0x63, 0xf2, 0x9e, 0xa4, 0x9c, 0xb2, 0x04, 0x7d, 0x01, 0xad, 0x2b, 0x1d, 0xda, 0x86, 0x6b, - 0x78, 0xd6, 0xf9, 0xf3, 0x71, 0xf1, 0xab, 0xf1, 0x7b, 0x12, 0x0a, 0x96, 0x4e, 0x1b, 0x77, 0xfb, - 0x61, 0x0d, 0x17, 0x6d, 0xe8, 0x0c, 0x9a, 0x11, 0xb9, 0xa2, 0x21, 0xb1, 0x9f, 0xb9, 0x86, 0xd7, - 0xc5, 0x79, 0x86, 0x6c, 0x68, 0xd1, 0xe4, 0xca, 0x8f, 0x69, 0x64, 0xd7, 0x5d, 0xc3, 0x6b, 0xe3, - 0x22, 0x95, 0x95, 0x88, 0xc4, 0x44, 0x90, 0xc8, 0x6e, 0xe8, 0x4a, 0x9e, 0x8e, 0x2e, 0xc0, 0xca, - 0x17, 0xf9, 0x9e, 0x72, 0x81, 0xbe, 0x84, 0x76, 0x3e, 0x85, 0xdb, 0x86, 0x5b, 0xf7, 0xac, 0xf3, - 0x8f, 0xc6, 0x51, 0x30, 0xae, 0xec, 0x9b, 0x2f, 0x53, 0xb6, 0x7d, 0xd5, 0xf8, 0xf5, 0x76, 0x58, - 0x1b, 0xfd, 0x66, 0xc2, 0x0b, 0xd9, 0x35, 0x4b, 0x96, 0xec, 0x5d, 0xba, 0x4b, 0x42, 0x5f, 0x90, - 0x08, 0x21, 0x68, 0x24, 0xfe, 0x86, 0xa8, 0xc3, 0x3a, 0x58, 0xc5, 0x12, 0xe3, 0xf4, 0x86, 0xa8, - 0x15, 0xeb, 0x58, 0xc5, 0xe8, 0x53, 0x80, 0x0d, 0x8b, 0xe8, 0x92, 0x92, 0x68, 0xc1, 0x6d, 0x53, - 0x55, 0x3a, 0x05, 0x32, 0x47, 0x1f, 0xc0, 0x2a, 0xcb, 0x41, 0x66, 0x77, 0x5d, 0xc3, 0x6b, 0x4c, - 0xbf, 0x96, 0x7b, 0xfc, 0xb1, 0x1f, 0xbe, 0x5e, 0x51, 0xb1, 0xde, 0x05, 0xe3, 0x90, 0x6d, 0x26, - 0x3c, 0x4b, 0x42, 0xb1, 0xa6, 0xc9, 0xaa, 0x12, 0x55, 0x65, 0x18, 0xcf, 0xd7, 0x2c, 0x15, 0xb3, - 0x37, 0xb8, 0x1c, 0x37, 0xcd, 0xaa, 0x02, 0x74, 0x9e, 0x26, 0xc0, 0x00, 0xda, 0x9c, 0xfc, 0xb4, - 0x23, 0x49, 0x48, 0x6c, 0x50, 0xcb, 0x96, 0x39, 0x7a, 0x05, 0x7d, 0x9e, 0x6d, 0x62, 0x9a, 0x5c, - 0x2e, 0x84, 0x9f, 0xae, 0x88, 0xb0, 0x5f, 0xa8, 0xe3, 0x7b, 0x39, 0xfa, 0x4e, 0x81, 0x68, 0x08, - 0x56, 0x10, 0xb3, 0xf0, 0x92, 0x2f, 0xd6, 0x3e, 0x5f, 0xdb, 0x48, 0x09, 0x09, 0x1a, 0xfa, 0xce, - 0xe7, 0x6b, 0xf4, 0x39, 0x34, 0x44, 0xb6, 0xd5, 0x12, 0xf7, 0xcf, 0xcf, 0x4e, 0x2b, 0x95, 0x2c, - 0x67, 0x5b, 0x82, 0x55, 0x0f, 0x72, 0xc1, 0xda, 0x92, 0x74, 0x43, 0xb9, 0x16, 0x4e, 0x4a, 0xdc, - 0xc3, 0x55, 0x48, 0x8e, 0x2b, 0x19, 0x4c, 0xb8, 0x6d, 0xb9, 0x86, 0x67, 0x9e, 0x48, 0xf8, 0x81, - 0xa3, 0x09, 0xe8, 0xe1, 0x0b, 0xa5, 0x4d, 0x4f, 0xd6, 0xa7, 0xcf, 0x0f, 0xfb, 0x61, 0x17, 0xfb, - 0xd7, 0x53, 0x59, 0x98, 0xd3, 0x1b, 0x82, 0x3b, 0x41, 0x11, 0xca, 0x99, 0x31, 0x0b, 0xfd, 0x78, - 0xb1, 0x8c, 0xfd, 0x15, 0xb7, 0xff, 0x69, 0xa9, 0xa1, 0xa0, 0xb0, 0x0b, 0x09, 0xa1, 0x11, 0x74, - 0x73, 0xc2, 0xf4, 0x8d, 0xff, 0xb6, 0xd4, 0x91, 0x56, 0x0e, 0xaa, 0x2b, 0x2b, 0xc6, 0x6c, 0x3e, - 0x30, 0x26, 0xf2, 0x4e, 0x66, 0x96, 0xbf, 0x6b, 0x4f, 0xfb, 0x87, 0xfd, 0x10, 0xb0, 0x7f, 0x3d, - 0xd3, 0xe8, 0xc9, 0xdc, 0xaf, 0xa0, 0x9f, 0xb0, 0x45, 0x95, 0x80, 0xb6, 0xfa, 0xab, 0x5e, 0xc2, - 0x7e, 0x3c, 0x81, 0xb9, 0x4f, 0xbf, 0x81, 0x8e, 0x3a, 0x27, 0x77, 0x7b, 0x53, 0x25, 0x85, 0xd7, - 0x3f, 0x3e, 0xb1, 0xac, 0x70, 0x49, 0x73, 0xae, 0x7d, 0xde, 0x38, 0xfa, 0x00, 0x9f, 0xcc, 0x92, - 0x88, 0xa6, 0x24, 0x14, 0xf9, 0x0d, 0x84, 0xbf, 0x4d, 0xe2, 0xec, 0xff, 0x05, 0x7d, 0x02, 0x1d, - 0xa3, 0xbf, 0x0d, 0x68, 0x7e, 0xcb, 0x76, 0x89, 0xe0, 0xe8, 0x25, 0x98, 0x4b, 0x1a, 0x13, 0xae, - 0xbe, 0x1d, 0x13, 0xeb, 0x44, 0xb2, 0xae, 0x87, 0xb3, 0x94, 0x12, 0xae, 0xcc, 0x61, 0xe2, 0x2a, - 0xa4, 0xbc, 0xa9, 0x9d, 0xc6, 0xd5, 0x27, 0x66, 0xe2, 0x32, 0x7f, 0xfc, 0x0c, 0x98, 0x27, 0xb6, - 0x5f, 0x82, 0x19, 0x64, 0x82, 0x14, 0xdf, 0x9e, 0x4e, 0x1e, 0xf8, 0xbc, 0xf9, 0xc8, 0xe7, 0x03, - 0x68, 0xeb, 0x67, 0x67, 0xf6, 0x46, 0x39, 0xbc, 0x8b, 0xcb, 0x1c, 0x39, 0x50, 0xf1, 0x81, 0xa2, - 0xe2, 0x81, 0x33, 0x46, 0x6f, 0xa1, 0xa3, 0xaf, 0x9c, 0x13, 0x81, 0x3c, 0x68, 0x86, 0x2a, 0xc9, - 0x45, 0x00, 0xf9, 0xe0, 0xe8, 0x72, 0xc1, 0xbd, 0xae, 0xcb, 0xf5, 0xc3, 0x94, 0xc8, 0x87, 0x45, - 0x1d, 0x5e, 0xc7, 0x45, 0x3a, 0x75, 0xef, 0xfe, 0x72, 0x6a, 0x77, 0x07, 0xc7, 0xb8, 0x3f, 0x38, - 0xc6, 0x9f, 0x07, 0xa7, 0xf6, 0xcb, 0xd1, 0xa9, 0xdd, 0x1e, 0x1d, 0xe3, 0xfe, 0xe8, 0xd4, 0x7e, - 0x3f, 0x3a, 0xb5, 0xa0, 0xa9, 0x94, 0x7d, 0xfd, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3a, 0x45, - 0x1c, 0xc5, 0xce, 0x05, 0x00, 0x00, + // 861 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4f, 0x8f, 0xdb, 0x54, + 0x10, 0x8f, 0x9b, 0xff, 0xe3, 0x64, 0xdb, 0xbe, 0x76, 0x57, 0x66, 0x25, 0x1c, 0xcb, 0x08, 0x61, + 0x71, 0x48, 0x60, 0x7b, 0xa3, 0x12, 0x42, 0x61, 0x55, 0x11, 0x09, 0x51, 0xf4, 0xb6, 0xf4, 0x80, + 0x2a, 0x45, 0xb6, 0xf3, 0x92, 0x3c, 0xd5, 0xf1, 0x0b, 0x7e, 0xce, 0xae, 0xdc, 0x4f, 0xc1, 0x05, + 0x89, 0x03, 0x87, 0x5e, 0xf8, 0x2e, 0x7b, 0xec, 0x11, 0x71, 0x88, 0x20, 0xcb, 0x01, 0xbe, 0x05, + 0x7a, 0xf3, 0x6c, 0xc7, 0x1b, 0x0e, 0xb4, 0xb7, 0x99, 0xdf, 0xcc, 0xf3, 0xcc, 0xfc, 0xe6, 0xe7, + 0x81, 0xbe, 0x4c, 0x93, 0x4d, 0x98, 0xca, 0xe1, 0x3a, 0x11, 0xa9, 0x20, 0x77, 0x66, 0xc1, 0xe9, + 0x07, 0x09, 0x5b, 0x0b, 0x39, 0x42, 0x20, 0xd8, 0xcc, 0x47, 0x0b, 0xb1, 0x10, 0xe8, 0xa0, 0xa5, + 0x13, 0x4f, 0x4f, 0x22, 0x1e, 0xe8, 0x94, 0x50, 0x44, 0xa3, 0x80, 0xad, 0x35, 0xee, 0xfe, 0x62, + 0x80, 0xf9, 0x84, 0x47, 0xec, 0x39, 0x4b, 0x24, 0x17, 0x31, 0xf9, 0x04, 0xda, 0x97, 0xda, 0xb4, + 0x0c, 0xc7, 0xf0, 0xcc, 0xb3, 0x7b, 0xc3, 0xe2, 0xd5, 0xf0, 0x39, 0x0b, 0x53, 0x91, 0x8c, 0x1b, + 0xd7, 0xdb, 0x41, 0x8d, 0x16, 0x69, 0xc4, 0x82, 0xf6, 0x8c, 0x45, 0x2c, 0x65, 0x33, 0xeb, 0x8e, + 0x63, 0x78, 0x1d, 0x5a, 0xb8, 0x3a, 0x72, 0xc9, 0x43, 0x26, 0xad, 0xba, 0x53, 0xf7, 0x7a, 0xb4, + 0x70, 0xc9, 0x47, 0x70, 0x97, 0xc7, 0x97, 0x7e, 0xc4, 0x67, 0xd3, 0x22, 0xa3, 0x81, 0x19, 0x47, + 0x39, 0x7c, 0xae, 0x51, 0xf7, 0x3b, 0x30, 0xf3, 0xce, 0xbe, 0xe6, 0x32, 0x25, 0x5f, 0x40, 0x27, + 0x2f, 0x2b, 0x2d, 0xc3, 0xa9, 0x7b, 0xe6, 0xd9, 0xdd, 0xe1, 0x2c, 0x18, 0x56, 0x06, 0x18, 0x3f, + 0x50, 0xdd, 0xed, 0xb6, 0x03, 0x93, 0xfa, 0x57, 0x39, 0x26, 0x69, 0xf9, 0xea, 0xb3, 0xc6, 0xcf, + 0xaf, 0x07, 0x35, 0xf7, 0xd7, 0x26, 0xdc, 0x57, 0x8f, 0x26, 0xf1, 0x5c, 0x3c, 0x4b, 0x36, 0x71, + 0xe8, 0xab, 0x7e, 0x09, 0x34, 0x62, 0x7f, 0xc5, 0x70, 0xf0, 0x2e, 0x45, 0x5b, 0x61, 0x92, 0xbf, + 0x62, 0x56, 0xdd, 0x31, 0xbc, 0x3a, 0x45, 0x9b, 0xbc, 0x0f, 0xb0, 0x12, 0x33, 0x3e, 0xe7, 0x6c, + 0x36, 0x95, 0x56, 0x13, 0x23, 0xdd, 0x02, 0xb9, 0x20, 0x2f, 0xc0, 0x2c, 0xc3, 0x41, 0x66, 0xf5, + 0x1c, 0xc3, 0x6b, 0x8c, 0x1f, 0xab, 0xb6, 0x7e, 0xdf, 0x0e, 0x1e, 0x2d, 0x78, 0xba, 0xdc, 0x04, + 0xc3, 0x50, 0xac, 0x46, 0x32, 0x8b, 0xc3, 0x74, 0xc9, 0xe3, 0x45, 0xc5, 0xaa, 0xae, 0x69, 0x78, + 0xb1, 0x14, 0x49, 0x3a, 0x39, 0xa7, 0x65, 0xb9, 0x71, 0x56, 0x5d, 0x50, 0xf7, 0xed, 0x16, 0x74, + 0x0a, 0x1d, 0xc9, 0x7e, 0xd8, 0xb0, 0x38, 0x64, 0x16, 0x60, 0xb3, 0xa5, 0x4f, 0x3e, 0x84, 0x23, + 0x99, 0xad, 0x22, 0x1e, 0xbf, 0x9c, 0xa6, 0x7e, 0xb2, 0x60, 0xa9, 0x75, 0x1f, 0x87, 0xef, 0xe7, + 0xe8, 0x33, 0x04, 0xc9, 0x00, 0xcc, 0x20, 0x12, 0xe1, 0x4b, 0x39, 0x5d, 0xfa, 0x72, 0x69, 0x11, + 0xc7, 0xf0, 0x7a, 0x14, 0x34, 0xf4, 0x95, 0x2f, 0x97, 0xe4, 0x63, 0x68, 0xa4, 0xd9, 0x9a, 0xa1, + 0x02, 0x8e, 0xce, 0x4e, 0xf6, 0x2d, 0x95, 0x2c, 0x67, 0x6b, 0x46, 0x31, 0x87, 0x38, 0x60, 0xae, + 0x59, 0xb2, 0xe2, 0x52, 0xef, 0xb1, 0xe1, 0x18, 0x5e, 0x9f, 0x56, 0x21, 0x55, 0xae, 0x64, 0x30, + 0x96, 0x96, 0xe9, 0x18, 0x5e, 0x73, 0x4f, 0xc2, 0x37, 0x92, 0x8c, 0x40, 0x17, 0x9f, 0xe2, 0x6e, + 0xfa, 0x2a, 0x3e, 0xbe, 0xb7, 0xdb, 0x0e, 0x7a, 0xd4, 0xbf, 0x1a, 0xab, 0xc0, 0x05, 0x7f, 0xc5, + 0x68, 0x37, 0x28, 0x4c, 0x55, 0x33, 0x12, 0xa1, 0x1f, 0x4d, 0xe7, 0x91, 0xbf, 0x90, 0xd6, 0xdf, + 0x6d, 0x2c, 0x0a, 0x88, 0x3d, 0x51, 0x10, 0x71, 0xa1, 0x97, 0x13, 0xa6, 0x67, 0xfc, 0xa7, 0x8d, + 0x43, 0x9a, 0x39, 0x88, 0x53, 0x56, 0xa4, 0xde, 0xba, 0x2d, 0x75, 0x0f, 0xda, 0xb9, 0x72, 0x2d, + 0xf5, 0xae, 0x33, 0x3e, 0xda, 0x6d, 0x07, 0x40, 0xfd, 0xab, 0x89, 0x46, 0x69, 0x11, 0x56, 0x8c, + 0xc7, 0x62, 0x5a, 0x25, 0xa0, 0x83, 0x9f, 0xea, 0xc7, 0xe2, 0xdb, 0x3d, 0x98, 0xeb, 0xf4, 0x73, + 0xe8, 0xe2, 0x38, 0x28, 0xfe, 0x4f, 0xa1, 0x85, 0x4e, 0x21, 0xfd, 0x07, 0x7b, 0x96, 0x11, 0x57, + 0x34, 0xe7, 0xbb, 0xcf, 0x13, 0xdd, 0x17, 0x70, 0x3c, 0x89, 0x67, 0x3c, 0x61, 0x61, 0x9a, 0xcf, + 0xc0, 0xe4, 0xd3, 0x38, 0xca, 0xfe, 0x7f, 0xa1, 0x6f, 0x41, 0x87, 0xfb, 0x97, 0x01, 0xad, 0x2f, + 0xc5, 0x26, 0x4e, 0x25, 0x79, 0x08, 0xcd, 0x39, 0x8f, 0x98, 0xc4, 0x7f, 0xa7, 0x49, 0xb5, 0xa3, + 0x58, 0xd7, 0xc5, 0x45, 0xc2, 0x99, 0x44, 0x71, 0x34, 0x69, 0x15, 0x42, 0x6d, 0x6a, 0xa5, 0x49, + 0xfc, 0xc5, 0x9a, 0xb4, 0xf4, 0xab, 0x6c, 0x37, 0x30, 0x54, 0xb2, 0xfd, 0x10, 0x9a, 0x41, 0x96, + 0xb2, 0xe2, 0xdf, 0xd3, 0xce, 0x2d, 0x9d, 0xb7, 0x0e, 0x74, 0x7e, 0x0a, 0x1d, 0x7d, 0x68, 0x26, + 0xe7, 0xa8, 0xf0, 0x1e, 0x2d, 0x7d, 0x62, 0x43, 0x45, 0x07, 0x48, 0xc5, 0x2d, 0x65, 0xb8, 0x4f, + 0xa1, 0xab, 0xa7, 0xbc, 0x60, 0x29, 0xf1, 0xa0, 0x15, 0xa2, 0x93, 0x2f, 0x01, 0xd4, 0xfd, 0xd1, + 0xe1, 0x82, 0x7b, 0x1d, 0x57, 0xed, 0x87, 0x09, 0xf3, 0x8b, 0xbb, 0x58, 0xa7, 0x85, 0xeb, 0xfe, + 0x64, 0xc0, 0x71, 0xe5, 0x64, 0x9d, 0xb3, 0x75, 0xc2, 0xf4, 0x05, 0x7a, 0xf7, 0xeb, 0x7b, 0x02, + 0x2d, 0x3d, 0x08, 0x16, 0xe9, 0xd1, 0xdc, 0x53, 0xd5, 0x0b, 0x41, 0xd6, 0xb5, 0x54, 0x0b, 0x01, + 0x1e, 0xd0, 0xba, 0x17, 0xb1, 0xfb, 0x3d, 0x1c, 0x57, 0x8e, 0x6d, 0xa5, 0xad, 0xc7, 0xff, 0x39, + 0xbb, 0xef, 0x1d, 0x9c, 0xdd, 0x7d, 0x72, 0xde, 0xe0, 0xc1, 0xc5, 0x1d, 0x3b, 0xd7, 0x7f, 0xda, + 0xb5, 0xeb, 0x9d, 0x6d, 0xbc, 0xd9, 0xd9, 0xc6, 0x1f, 0x3b, 0xbb, 0xf6, 0xe3, 0x8d, 0x5d, 0x7b, + 0x7d, 0x63, 0x1b, 0x6f, 0x6e, 0xec, 0xda, 0x6f, 0x37, 0x76, 0x2d, 0x68, 0xe1, 0xa4, 0x8f, 0xfe, + 0x0d, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xe1, 0xbd, 0x08, 0xe2, 0x06, 0x00, 0x00, } func (m *FileVersion) Marshal() (dAtA []byte, err error) { @@ -403,6 +486,24 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.InvalidDevices) > 0 { + for iNdEx := len(m.InvalidDevices) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.InvalidDevices[iNdEx]) + copy(dAtA[i:], m.InvalidDevices[iNdEx]) + i = encodeVarintStructs(dAtA, i, uint64(len(m.InvalidDevices[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if len(m.Devices) > 0 { + for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Devices[iNdEx]) + copy(dAtA[i:], m.Devices[iNdEx]) + i = encodeVarintStructs(dAtA, i, uint64(len(m.Devices[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } if m.Deleted { i-- if m.Deleted { @@ -411,24 +512,7 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0 } i-- - dAtA[i] = 0x20 - } - if m.Invalid { - i-- - if m.Invalid { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i-- - dAtA[i] = 0x18 - } - if len(m.Device) > 0 { - i -= len(m.Device) - copy(dAtA[i:], m.Device) - i = encodeVarintStructs(dAtA, i, uint64(len(m.Device))) - i-- - dAtA[i] = 0x12 + dAtA[i] = 0x10 } { size, err := m.Version.MarshalToSizedBuffer(dAtA[:i]) @@ -463,10 +547,10 @@ func (m *VersionList) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Versions) > 0 { - for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- { + if len(m.RawVersions) > 0 { + for iNdEx := len(m.RawVersions) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.RawVersions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -813,6 +897,103 @@ func (m *CountsSet) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FileVersionDeprecated) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileVersionDeprecated) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FileVersionDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Deleted { + i-- + if m.Deleted { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } + if m.Invalid { + i-- + if m.Invalid { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x18 + } + if len(m.Device) > 0 { + i -= len(m.Device) + copy(dAtA[i:], m.Device) + i = encodeVarintStructs(dAtA, i, uint64(len(m.Device))) + i-- + dAtA[i] = 0x12 + } + { + size, err := m.Version.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStructs(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *VersionListDeprecated) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VersionListDeprecated) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VersionListDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Versions) > 0 { + for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintStructs(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func encodeVarintStructs(dAtA []byte, offset int, v uint64) int { offset -= sovStructs(v) base := offset @@ -832,16 +1013,21 @@ func (m *FileVersion) ProtoSize() (n int) { _ = l l = m.Version.ProtoSize() n += 1 + l + sovStructs(uint64(l)) - l = len(m.Device) - if l > 0 { - n += 1 + l + sovStructs(uint64(l)) - } - if m.Invalid { - n += 2 - } if m.Deleted { n += 2 } + if len(m.Devices) > 0 { + for _, b := range m.Devices { + l = len(b) + n += 1 + l + sovStructs(uint64(l)) + } + } + if len(m.InvalidDevices) > 0 { + for _, b := range m.InvalidDevices { + l = len(b) + n += 1 + l + sovStructs(uint64(l)) + } + } return n } @@ -851,8 +1037,8 @@ func (m *VersionList) ProtoSize() (n int) { } var l int _ = l - if len(m.Versions) > 0 { - for _, e := range m.Versions { + if len(m.RawVersions) > 0 { + for _, e := range m.RawVersions { l = e.ProtoSize() n += 1 + l + sovStructs(uint64(l)) } @@ -1007,6 +1193,42 @@ func (m *CountsSet) ProtoSize() (n int) { return n } +func (m *FileVersionDeprecated) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Version.ProtoSize() + n += 1 + l + sovStructs(uint64(l)) + l = len(m.Device) + if l > 0 { + n += 1 + l + sovStructs(uint64(l)) + } + if m.Invalid { + n += 2 + } + if m.Deleted { + n += 2 + } + return n +} + +func (m *VersionListDeprecated) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Versions) > 0 { + for _, e := range m.Versions { + l = e.ProtoSize() + n += 1 + l + sovStructs(uint64(l)) + } + } + return n +} + func sovStructs(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -1076,8 +1298,28 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Deleted = bool(v != 0) + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1104,36 +1346,14 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...) - if m.Device == nil { - m.Device = []byte{} - } + m.Devices = append(m.Devices, make([]byte, postIndex-iNdEx)) + copy(m.Devices[len(m.Devices)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowStructs - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Invalid = bool(v != 0) case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InvalidDevices", wireType) } - var v int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowStructs @@ -1143,12 +1363,24 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - m.Deleted = bool(v != 0) + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.InvalidDevices = append(m.InvalidDevices, make([]byte, postIndex-iNdEx)) + copy(m.InvalidDevices[len(m.InvalidDevices)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipStructs(dAtA[iNdEx:]) @@ -1204,7 +1436,7 @@ func (m *VersionList) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RawVersions", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1231,8 +1463,8 @@ func (m *VersionList) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Versions = append(m.Versions, FileVersion{}) - if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.RawVersions = append(m.RawVersions, FileVersion{}) + if err := m.RawVersions[len(m.RawVersions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -2243,6 +2475,253 @@ func (m *CountsSet) Unmarshal(dAtA []byte) error { } return nil } +func (m *FileVersionDeprecated) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileVersionDeprecated: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileVersionDeprecated: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Version.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...) + if m.Device == nil { + m.Device = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Invalid = bool(v != 0) + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Deleted = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VersionListDeprecated) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VersionListDeprecated: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VersionListDeprecated: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStructs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthStructs + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthStructs + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Versions = append(m.Versions, FileVersionDeprecated{}) + if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipStructs(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthStructs + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipStructs(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/lib/db/structs.proto b/lib/db/structs.proto index 5a488e9a8..f21b27cd8 100644 --- a/lib/db/structs.proto +++ b/lib/db/structs.proto @@ -13,15 +13,15 @@ option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.goproto_sizecache_all) = false; message FileVersion { - protocol.Vector version = 1 [(gogoproto.nullable) = false]; - bytes device = 2; - bool invalid = 3; - bool deleted = 4; + protocol.Vector version = 1 [(gogoproto.nullable) = false]; + bool deleted = 2; + repeated bytes devices = 3; + repeated bytes invalid_devices = 4; } message VersionList { option (gogoproto.goproto_stringer) = false; - repeated FileVersion versions = 1 [(gogoproto.nullable) = false]; + repeated FileVersion versions = 1 [(gogoproto.customname) = "RawVersions", (gogoproto.nullable) = false]; } // Must be the same as FileInfo but without the blocks field @@ -79,3 +79,15 @@ message CountsSet { repeated Counts counts = 1 [(gogoproto.nullable) = false]; int64 created = 2; // unix nanos } + +message FileVersionDeprecated { + protocol.Vector version = 1 [(gogoproto.nullable) = false]; + bytes device = 2; + bool invalid = 3; + bool deleted = 4; +} + +message VersionListDeprecated { + option (gogoproto.goproto_stringer) = false; + repeated FileVersionDeprecated versions = 1 [(gogoproto.nullable) = false]; +} diff --git a/lib/db/transactions.go b/lib/db/transactions.go index 4660247a3..05cacbaac 100644 --- a/lib/db/transactions.go +++ b/lib/db/transactions.go @@ -15,7 +15,11 @@ import ( "github.com/syncthing/syncthing/lib/protocol" ) -var errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry") +var ( + errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry") + errEmptyGlobal = errors.New("no versions in global list") + errEmptyFileVersion = errors.New("no devices in global file version") +) // A readOnlyTransaction represents a database snapshot. type readOnlyTransaction struct { @@ -54,7 +58,7 @@ func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool, return f.(protocol.FileInfo), true, nil } -func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, bool, error) { +func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileIntf, bool, error) { bs, err := t.Get(key) if backend.IsNotFound(err) { return nil, false, nil @@ -72,7 +76,7 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo return f, true, nil } -func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) { +func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileIntf, error) { if trunc { var tf FileInfoTruncated err := tf.Unmarshal(bs) @@ -175,26 +179,44 @@ func (t readOnlyTransaction) getGlobalVersionsByKey(key []byte) (VersionList, er return vl, nil } -func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) { +func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, protocol.FileIntf, bool, error) { vl, err := t.getGlobalVersions(keyBuf, folder, file) if backend.IsNotFound(err) { return keyBuf, nil, false, nil } else if err != nil { return nil, nil, false, err } - if len(vl.Versions) == 0 { - return nil, nil, false, nil - } + var fi protocol.FileIntf + keyBuf, fi, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, truncate, vl) + return keyBuf, fi, true, err +} - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file) +func (t readOnlyTransaction) getGlobalFromVersionList(keyBuf, folder, file []byte, truncate bool, vl VersionList) ([]byte, protocol.FileIntf, FileVersion, error) { + fv, ok := vl.GetGlobal() + if !ok { + return keyBuf, nil, FileVersion{}, errEmptyGlobal + } + keyBuf, fi, err := t.getGlobalFromFileVersion(keyBuf, folder, file, truncate, fv) + return keyBuf, fi, fv, err +} + +func (t readOnlyTransaction) getGlobalFromFileVersion(keyBuf, folder, file []byte, truncate bool, fv FileVersion) ([]byte, protocol.FileIntf, error) { + dev, ok := fv.FirstDevice() + if !ok { + return keyBuf, nil, errEmptyFileVersion + } + keyBuf, err := t.keyer.GenerateDeviceFileKey(keyBuf, folder, dev, file) if err != nil { - return nil, nil, false, err + return keyBuf, nil, err } fi, ok, err := t.getFileTrunc(keyBuf, truncate) - if err != nil || !ok { - return keyBuf, nil, false, err + if err != nil { + return keyBuf, nil, err } - return keyBuf, fi, true, nil + if !ok { + return keyBuf, nil, errEntryFromGlobalMissing + } + return keyBuf, fi, nil } func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error { @@ -320,19 +342,12 @@ func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, f return err } - dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name) + var f protocol.FileIntf + dk, f, _, err = t.getGlobalFromVersionList(dk, folder, name, truncate, vl) if err != nil { return err } - f, ok, err := t.getFileTrunc(dk, truncate) - if err != nil { - return err - } - if !ok { - continue - } - if !fn(f) { return nil } @@ -393,16 +408,13 @@ func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.Devi return nil, err } - var devices []protocol.DeviceID - for _, v := range vl.Versions { - if !v.Version.Equal(vl.Versions[0].Version) { - break - } - if v.Invalid { - continue - } - n := protocol.DeviceIDFromBytes(v.Device) - devices = append(devices, n) + fv, ok := vl.GetGlobal() + if !ok { + return nil, nil + } + devices := make([]protocol.DeviceID, len(fv.Devices)) + for i, dev := range fv.Devices { + devices[i] = protocol.DeviceIDFromBytes(dev) } return devices, nil @@ -431,25 +443,28 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn return err } - globalFV := vl.Versions[0] + globalFV, ok := vl.GetGlobal() + if !ok { + return errEmptyGlobal + } haveFV, have := vl.Get(device) if !need(globalFV, have, haveFV.Version) { continue } + name := t.keyer.NameFromGlobalVersionKey(dbi.Key()) - dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, globalFV.Device, name) - if err != nil { - return err - } - gf, ok, err := t.getFileTrunc(dk, truncate) + var gf protocol.FileIntf + dk, gf, err = t.getGlobalFromFileVersion(dk, folder, name, truncate, globalFV) if err != nil { return err } + + globalDev, ok := globalFV.FirstDevice() if !ok { - return errEntryFromGlobalMissing + return errEmptyFileVersion } - l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, globalFV.Version, globalFV.Device) + l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalDev) if !fn(gf) { return dbi.Error() } @@ -469,7 +484,7 @@ func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Ite defer dbi.Release() var keyBuf []byte - var f FileIntf + var f protocol.FileIntf var ok bool for dbi.Next() { keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate) @@ -586,7 +601,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi return nil, false, err } - fl, removedFV, removedAt, insertedAt, err := fl.update(folder, device, file, t.readOnlyTransaction) + globalFV, oldGlobalFV, removedFV, haveOldGlobal, haveRemoved, globalChanged, err := fl.update(folder, device, file, t.readOnlyTransaction) if err != nil { return nil, false, err } @@ -601,26 +616,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi // Only load those from db if actually needed var gotGlobal, gotOldGlobal bool - var global, oldGlobal FileIntf - - globalFV := fl.Versions[0] - var oldGlobalFV FileVersion - haveOldGlobal := false - - globalUnaffected := removedAt != 0 && insertedAt != 0 - if globalUnaffected { - oldGlobalFV = globalFV - haveOldGlobal = true - } else { - if removedAt == 0 { - oldGlobalFV = removedFV - haveOldGlobal = true - } else if len(fl.Versions) > 1 { - // The previous newest version is now at index 1 - oldGlobalFV = fl.Versions[1] - haveOldGlobal = true - } - } + var global, oldGlobal protocol.FileIntf // Check the need of the device that was updated // Must happen before updating global meta: If this is the first @@ -628,11 +624,11 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi needBefore := false if haveOldGlobal { - needBefore = need(oldGlobalFV, removedAt >= 0, removedFV.Version) + needBefore = need(oldGlobalFV, haveRemoved, removedFV.Version) } - needNow := need(globalFV, true, fl.Versions[insertedAt].Version) + needNow := need(globalFV, true, file.Version) if needBefore { - if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil { + if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil { return nil, false, err } gotOldGlobal = true @@ -644,7 +640,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi } } if needNow { - if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil { + if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil { return nil, false, err } gotGlobal = true @@ -657,42 +653,34 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi } // Update global size counter if necessary - // Necessary here means the first item in the global list was changed, - // even if both new and old are invalid, due to potential change in - // LocalFlags. - // Neither the global state nor the needs of any devices, except the one - // updated, changed. - if globalUnaffected { + if !globalChanged { + // Neither the global state nor the needs of any devices, except + // the one updated, changed. return keyBuf, true, nil } // Remove the old global from the global size counter if haveOldGlobal { if !gotOldGlobal { - if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil { + if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil { return nil, false, err } gotOldGlobal = true } + // Remove the old global from the global size counter meta.removeFile(protocol.GlobalDeviceID, oldGlobal) } // Add the new global to the global size counter if !gotGlobal { - if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil { + if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil { return nil, false, err } gotGlobal = true } meta.addFile(protocol.GlobalDeviceID, global) - // If global changed, but both the new and old are invalid, noone needed - // the file before and now -> nothing to do. - if global.IsInvalid() && (!haveOldGlobal || oldGlobal.IsInvalid()) { - return keyBuf, true, nil - } - // check for local (if not already done before) if !bytes.Equal(device, protocol.LocalDeviceID[:]) { localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:]) @@ -736,40 +724,12 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi return keyBuf, true, nil } -func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, insertedAt int, fl VersionList) (FileIntf, error) { - if insertedAt == 0 { +func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, fv FileVersion) ([]byte, protocol.FileIntf, error) { + if fv.Version.Equal(file.Version) { // Inserted a new newest version - return file, nil + return keyBuf, file, nil } - var err error - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name) - if err != nil { - return nil, err - } - global, ok, err := t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, err - } - if !ok { - return nil, errEntryFromGlobalMissing - } - return global, nil -} - -func (t readWriteTransaction) updateGlobalGetOldGlobal(keyBuf, folder, name []byte, oldGlobalFV FileVersion) (FileIntf, error) { - var err error - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name) - if err != nil { - return nil, err - } - oldGlobal, ok, err := t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, err - } - if !ok { - return nil, errEntryFromGlobalMissing - } - return oldGlobal, nil + return t.getGlobalFromFileVersion(keyBuf, folder, name, true, fv) } func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add bool) ([]byte, error) { @@ -790,7 +750,7 @@ func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add b func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool { // We never need an invalid file. - if global.Invalid { + if global.IsInvalid() { return false } // We don't need a deleted file if we don't have it. @@ -807,7 +767,7 @@ func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool // removeFromGlobal removes the device from the global version list for the // given file. If the version list is empty after this, the file entry is // removed entirely. -func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) { +func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file []byte, meta *metadataTracker) ([]byte, error) { deviceID := protocol.DeviceIDFromBytes(device) l.Debugf("remove from global; folder=%q device=%v file=%q", folder, deviceID, file) @@ -821,34 +781,32 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte return nil, err } - if len(fl.Versions) == 0 { + oldGlobalFV, haveOldGlobal := fl.GetGlobal() + + if !haveOldGlobal { // Shouldn't ever happen, but doesn't hurt to handle. return keyBuf, t.Delete(gk) } - oldGlobalFV := fl.Versions[0] - - fl, removedFV, removedAt := fl.pop(device) - if removedAt == -1 { + removedFV, haveRemoved, globalChanged, err := fl.pop(folder, device, file, t.readOnlyTransaction) + if err != nil { + return nil, err + } + if !haveRemoved { // There is no version for the given device return keyBuf, nil } - var global FileIntf + var global protocol.FileIntf var gotGlobal, ok bool + globalFV, ok := fl.GetGlobal() // Add potential needs of the removed device - if len(fl.Versions) != 0 && !fl.Versions[0].Invalid && need(fl.Versions[0], false, protocol.Vector{}) && !need(oldGlobalFV, removedAt != -1, removedFV.Version) { - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file) + if ok && !globalFV.IsInvalid() && need(globalFV, false, protocol.Vector{}) && !need(oldGlobalFV, haveRemoved, removedFV.Version) { + keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl) if err != nil { return nil, err } - global, ok, err = t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, err - } else if !ok { - return nil, errEntryFromGlobalMissing - } gotGlobal = true meta.addNeeded(deviceID, global) if bytes.Equal(protocol.LocalDeviceID[:], device) { @@ -859,7 +817,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte } // Global hasn't changed, abort early - if removedAt != 0 { + if !globalChanged { l.Debugf("new global after remove: %v", fl) if err := t.Put(gk, mustMarshal(&fl)); err != nil { return nil, err @@ -896,7 +854,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte } // Nothing left, i.e. nothing to add to the global counter below. - if len(fl.Versions) == 0 { + if fl.Empty() { if err := t.Delete(gk); err != nil { return nil, err } @@ -905,21 +863,14 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte // Add to global if !gotGlobal { - keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file) + keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl) if err != nil { return nil, err } - global, ok, err = t.getFileTrunc(keyBuf, true) - if err != nil { - return nil, err - } - if !ok { - return nil, errEntryFromGlobalMissing - } } meta.addFile(protocol.GlobalDeviceID, global) - l.Debugf("new global after remove: %v", fl) + l.Debugf(`new global for "%s" after remove: %v`, file, fl) if err := t.Put(gk, mustMarshal(&fl)); err != nil { return nil, err } diff --git a/lib/model/folder.go b/lib/model/folder.go index dea9b4950..8f1692542 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -303,7 +303,7 @@ func (f *folder) pull() (success bool) { // If there is nothing to do, don't even enter sync-waiting state. abort := true snap := f.fset.Snapshot() - snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { abort = false return false }) @@ -499,7 +499,7 @@ func (f *folder) scanSubdirs(subDirs []string) error { for _, sub := range subDirs { var iterError error - snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool { + snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi protocol.FileIntf) bool { select { case <-f.ctx.Done(): return false @@ -634,7 +634,7 @@ func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file proto found := false nf := protocol.FileInfo{} - snap.WithBlocksHash(file.BlocksHash, func(ifi db.FileIntf) bool { + snap.WithBlocksHash(file.BlocksHash, func(ifi protocol.FileIntf) bool { fi := ifi.(protocol.FileInfo) select { diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go index 81b4068ac..85ced3431 100644 --- a/lib/model/folder_recvonly.go +++ b/lib/model/folder_recvonly.go @@ -87,7 +87,7 @@ func (f *receiveOnlyFolder) revert() { batchSizeBytes := 0 snap := f.fset.Snapshot() defer snap.Release() - snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { fi := intf.(protocol.FileInfo) if !fi.IsReceiveOnlyChanged() { // We're only interested in files that have changed locally in diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index 18c4ac9c5..17fb7c321 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -52,7 +52,7 @@ func (f *sendOnlyFolder) pull() bool { snap := f.fset.Snapshot() defer snap.Release() - snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { f.updateLocalsFromPulling(batch) batch = batch[:0] @@ -110,7 +110,7 @@ func (f *sendOnlyFolder) override() { batchSizeBytes := 0 snap := f.fset.Snapshot() defer snap.Release() - snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool { need := fi.(protocol.FileInfo) if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes { f.updateLocalsFromScanning(batch) diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index a657a71bc..64ec8a654 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -305,7 +305,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<- // Regular files to pull goes into the file queue, everything else // (directories, symlinks and deletes) goes into the "process directly" // pile. - snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool { + snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool { select { case <-f.ctx.Done(): return false diff --git a/lib/model/model.go b/lib/model/model.go index 6f8aaf82b..68c63fe75 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -851,7 +851,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo } rest = make([]db.FileInfoTruncated, 0, perpage) - snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool { + snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { if cfg.IgnoreDelete && f.IsDeleted() { return true } @@ -1936,7 +1936,7 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error { snap := s.fset.Snapshot() defer snap.Release() previousWasDelete := false - snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool { + snap.WithHaveSequence(s.prevSequence+1, func(fi protocol.FileIntf) bool { // This is to make sure that renames (which is an add followed by a delete) land in the same batch. // Even if the batch is full, we allow a last delete to slip in, we do this by making sure that // the batch ends with a non-delete, or that the last item in the batch is already a delete @@ -2248,7 +2248,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly snap := files.Snapshot() defer snap.Release() - snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool { + snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool { f := fi.(db.FileInfoTruncated) // Don't include the prefix itself. diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 7be967562..caae11e53 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -2426,7 +2426,7 @@ func TestIssue3496(t *testing.T) { m.fmut.RUnlock() var localFiles []protocol.FileInfo snap := fs.Snapshot() - snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool { + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { localFiles = append(localFiles, i.(protocol.FileInfo)) return true }) @@ -3556,7 +3556,7 @@ func TestRenameSequenceOrder(t *testing.T) { count := 0 snap := dbSnapshot(t, m, "default") - snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool { + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { count++ return true }) @@ -3588,7 +3588,7 @@ func TestRenameSequenceOrder(t *testing.T) { var firstExpectedSequence int64 var secondExpectedSequence int64 failed := false - snap.WithHaveSequence(0, func(i db.FileIntf) bool { + snap.WithHaveSequence(0, func(i protocol.FileIntf) bool { t.Log(i) if i.FileName() == "17" { firstExpectedSequence = i.SequenceNo() + 1 @@ -3621,7 +3621,7 @@ func TestRenameSameFile(t *testing.T) { count := 0 snap := dbSnapshot(t, m, "default") - snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool { + snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool { count++ return true }) @@ -3644,7 +3644,7 @@ func TestRenameSameFile(t *testing.T) { prevSeq := int64(0) seen := false - snap.WithHaveSequence(0, func(i db.FileIntf) bool { + snap.WithHaveSequence(0, func(i protocol.FileIntf) bool { if i.SequenceNo() <= prevSeq { t.Fatalf("non-increasing sequences: %d <= %d", i.SequenceNo(), prevSeq) } @@ -3683,7 +3683,7 @@ func TestRenameEmptyFile(t *testing.T) { } count := 0 - snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool { + snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool { count++ return true }) @@ -3693,7 +3693,7 @@ func TestRenameEmptyFile(t *testing.T) { } count = 0 - snap.WithBlocksHash(file.BlocksHash, func(_ db.FileIntf) bool { + snap.WithBlocksHash(file.BlocksHash, func(_ protocol.FileIntf) bool { count++ return true }) @@ -3712,7 +3712,7 @@ func TestRenameEmptyFile(t *testing.T) { defer snap.Release() count = 0 - snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool { + snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool { count++ return true }) @@ -3722,7 +3722,7 @@ func TestRenameEmptyFile(t *testing.T) { } count = 0 - snap.WithBlocksHash(file.BlocksHash, func(i db.FileIntf) bool { + snap.WithBlocksHash(file.BlocksHash, func(i protocol.FileIntf) bool { count++ if i.FileName() != "new-file" { t.Fatalf("unexpected file name %s, expected new-file", i.FileName()) @@ -3757,7 +3757,7 @@ func TestBlockListMap(t *testing.T) { } var paths []string - snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool { + snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool { paths = append(paths, fi.FileName()) return true }) @@ -3790,7 +3790,7 @@ func TestBlockListMap(t *testing.T) { defer snap.Release() paths = paths[:0] - snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool { + snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool { paths = append(paths, fi.FileName()) return true }) diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index d37cd1c5e..b42b05073 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -23,6 +23,31 @@ const ( Version13HelloMagic uint32 = 0x9F79BC40 // old ) +// FileIntf is the set of methods implemented by both FileInfo and +// db.FileInfoTruncated. +type FileIntf interface { + FileSize() int64 + FileName() string + FileLocalFlags() uint32 + IsDeleted() bool + IsInvalid() bool + IsIgnored() bool + IsUnsupported() bool + MustRescan() bool + IsReceiveOnlyChanged() bool + IsDirectory() bool + IsSymlink() bool + ShouldConflict() bool + HasPermissionBits() bool + SequenceNo() int64 + BlockSize() int + FileVersion() Vector + FileType() FileInfoType + FilePermissions() uint32 + FileModifiedBy() ShortID + ModTime() time.Time +} + func (m Hello) Magic() uint32 { return HelloMessageMagic } @@ -139,7 +164,7 @@ func (f FileInfo) FileModifiedBy() ShortID { // WinsConflict returns true if "f" is the one to choose when it is in // conflict with "other". -func (f FileInfo) WinsConflict(other FileInfo) bool { +func WinsConflict(f, other FileIntf) bool { // If only one of the files is invalid, that one loses. if f.IsInvalid() != other.IsInvalid() { return !f.IsInvalid() @@ -164,7 +189,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool { // The modification times were equal. Use the device ID in the version // vector as tie breaker. - return f.Version.Compare(other.Version) == ConcurrentGreater + return f.FileVersion().Compare(other.FileVersion()) == ConcurrentGreater } func (f FileInfo) IsEmpty() bool { diff --git a/lib/protocol/conflict_test.go b/lib/protocol/conflict_test.go index b5318387f..a606c47e4 100644 --- a/lib/protocol/conflict_test.go +++ b/lib/protocol/conflict_test.go @@ -14,10 +14,10 @@ func TestWinsConflict(t *testing.T) { } for _, tc := range testcases { - if !tc[0].WinsConflict(tc[1]) { + if !WinsConflict(tc[0], tc[1]) { t.Errorf("%v should win over %v", tc[0], tc[1]) } - if tc[1].WinsConflict(tc[0]) { + if WinsConflict(tc[1], tc[0]) { t.Errorf("%v should not win over %v", tc[1], tc[0]) } }