lib/db: Refactor to use global list by version (fixes #6372) (#6638)

Group the global list of files by version, instead of having one flat list for all devices. This removes lots of duplicate protocol.Vectors.

Co-authored-by: Jakob Borg <jakob@kastelo.net>

cmd/stindex/idxck.go

@@ -219,10 +219,10 @@ func idxck(ldb backend.Backend) (success bool) {
 			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
 			success = false
 		}
-		for i, fv := range vl.Versions {
+		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(fv.Device)]
+			dev, ok := deviceToIDs[string(device)]
 			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
 				success = false
 			}
 			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@@ -235,14 +235,26 @@ func idxck(ldb backend.Backend) (success bool) {
 			if fi.VersionHash != nil {
 				fiv = versions[string(fi.VersionHash)]
 			}
-			if !fiv.Equal(fv.Version) {
+			if !fiv.Equal(version) {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
 				success = false
 			}
-			if fi.IsInvalid() != fv.Invalid {
+			if fi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
 				success = false
 			}
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
+				success = false
+			}
+		}
+		for i, fv := range vl.RawVersions {
+			for _, device := range fv.Devices {
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
+			}
+			for _, device := range fv.InvalidDevices {
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
+			}
 		}
 
 		// If we need this file we should have a need entry for it. False
@@ -251,7 +263,9 @@ func idxck(ldb backend.Backend) (success bool) {
 		if needsLocally(vl) {
 			_, ok := needs[gk]
 			if !ok {
-				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
+				dev := deviceToIDs[string(devB)]
 				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
 				if !fi.IsDeleted() && !fi.IsIgnored() {
 					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
@@ -319,15 +333,10 @@ func idxck(ldb backend.Backend) (success bool) {
 }
 
 func needsLocally(vl db.VersionList) bool {
-	var lv *protocol.Vector
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
-	for _, fv := range vl.Versions {
+	if !ok {
-		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
-			lv = &fv.Version
-			break
-		}
-	}
-	if lv == nil {
 		return true // proviosinally, it looks like we need the file
 	}
-	return !lv.GreaterEqual(vl.Versions[0].Version)
+	gfv, _ := vl.GetGlobal() // Can't not have a global if we got something above
+	return !fv.Version.GreaterEqual(gfv.Version)
 }

lib/api/api.go

@@ -1635,7 +1635,7 @@ func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m)
 }
 
-func fileIntfJSONMap(f db.FileIntf) map[string]interface{} {
+func fileIntfJSONMap(f protocol.FileIntf) map[string]interface{} {
 	out := map[string]interface{}{
 		"name":          f.FileName(),
 		"type":          f.FileType().String(),

lib/db/benchmark_test.go

@@ -187,7 +187,7 @@ func BenchmarkNeedHalf(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -211,7 +211,7 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := fset.Snapshot()
-		snap.WithNeed(remoteDevice0, func(fi db.FileIntf) bool {
+		snap.WithNeed(remoteDevice0, func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -232,7 +232,7 @@ func BenchmarkHave(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -253,7 +253,7 @@ func BenchmarkGlobal(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithGlobal(func(fi db.FileIntf) bool {
+		snap.WithGlobal(func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -274,7 +274,7 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap.WithNeedTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -295,7 +295,7 @@ func BenchmarkHaveTruncated(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+		snap.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})
@@ -316,7 +316,7 @@ func BenchmarkGlobalTruncated(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		count := 0
 		snap := benchS.Snapshot()
-		snap.WithGlobalTruncated(func(fi db.FileIntf) bool {
+		snap.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
 			count++
 			return true
 		})

lib/db/db_test.go

@@ -183,7 +183,9 @@ func TestUpdate0to3(t *testing.T) {
 		t.Error("File prefixed by '/' was not removed during transition to schema 1")
 	}
 
-	key, err := db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid))
+	var key []byte
+
+	key, err = db.keyer.GenerateGlobalVersionKey(nil, folder, []byte(invalid))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -201,7 +203,7 @@ func TestUpdate0to3(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer trans.Release()
-	_ = trans.withHaveSequence(folder, 0, func(fi FileIntf) bool {
+	_ = trans.withHaveSequence(folder, 0, func(fi protocol.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		l.Infoln(f)
 		if found {
@@ -228,12 +230,42 @@ func TestUpdate0to3(t *testing.T) {
 		haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
 		haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
 	}
+
 	trans, err = db.newReadOnlyTransaction()
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer trans.Release()
-	_ = trans.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
+
+	key, err = trans.keyer.GenerateNeedFileKey(nil, folder, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dbi, err := trans.NewPrefixIterator(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dbi.Release()
+
+	for dbi.Next() {
+		name := trans.keyer.NameFromGlobalVersionKey(dbi.Key())
+		key, err = trans.keyer.GenerateGlobalVersionKey(key, folder, name)
+		bs, err := trans.Get(key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var vl VersionListDeprecated
+		if err := vl.Unmarshal(bs); err != nil {
+			t.Fatal(err)
+		}
+		key, err = trans.keyer.GenerateDeviceFileKey(key, folder, vl.Versions[0].Device, name)
+		fi, ok, err := trans.getFileTrunc(key, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !ok {
+			t.Fatal("surprise missing global file", string(name), protocol.DeviceIDFromBytes(vl.Versions[0].Device))
+		}
 		e, ok := need[fi.FileName()]
 		if !ok {
 			t.Error("Got unexpected needed file:", fi.FileName())
@@ -243,8 +275,11 @@ func TestUpdate0to3(t *testing.T) {
 		if !f.IsEquivalentOptional(e, 0, true, true, 0) {
 			t.Errorf("Wrong needed file, got %v, expected %v", f, e)
 		}
-		return true
+	}
-	})
+	if dbi.Error() != nil {
+		t.Fatal(err)
+	}
+
 	for n := range need {
 		t.Errorf(`Missing needed file "%v"`, n)
 	}
@@ -467,7 +502,7 @@ func TestCheckGlobals(t *testing.T) {
 	}
 
 	// Clean up global entry of the now missing file
-	if err := db.checkGlobals([]byte(fs.folder), fs.meta); err != nil {
+	if err := db.checkGlobals([]byte(fs.folder)); err != nil {
 		t.Fatal(err)
 	}
 
@@ -525,7 +560,7 @@ func TestUpdateTo10(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, v := range vl.Versions {
+	for _, v := range vl.RawVersions {
 		if !v.Deleted {
 			t.Error("Unexpected undeleted global version for a")
 		}
@@ -535,10 +570,10 @@ func TestUpdateTo10(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !vl.Versions[0].Deleted {
+	if !vl.RawVersions[0].Deleted {
 		t.Error("vl.Versions[0] not deleted for b")
 	}
-	if vl.Versions[1].Deleted {
+	if vl.RawVersions[1].Deleted {
 		t.Error("vl.Versions[1] deleted for b")
 	}
 	// c
@@ -546,10 +581,10 @@ func TestUpdateTo10(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if vl.Versions[0].Deleted {
+	if vl.RawVersions[0].Deleted {
 		t.Error("vl.Versions[0] deleted for c")
 	}
-	if !vl.Versions[1].Deleted {
+	if !vl.RawVersions[1].Deleted {
 		t.Error("vl.Versions[1] not deleted for c")
 	}
 }

lib/db/lowlevel.go

@@ -426,7 +426,7 @@ func (db *Lowlevel) dropDeviceFolder(device, folder []byte, meta *metadataTracke
 	return t.Commit()
 }
 
-func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
+func (db *Lowlevel) checkGlobals(folder []byte) error {
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -444,9 +444,10 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
 	defer dbi.Release()
 
 	var dk []byte
+	ro := t.readOnlyTransaction
 	for dbi.Next() {
 		var vl VersionList
-		if err := vl.Unmarshal(dbi.Value()); err != nil || len(vl.Versions) == 0 {
+		if err := vl.Unmarshal(dbi.Value()); err != nil || vl.Empty() {
 			if err := t.Delete(dbi.Key()); err != nil {
 				return err
 			}
@@ -459,36 +460,28 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
 		// we find those and clear them out.
 
 		name := db.keyer.NameFromGlobalVersionKey(dbi.Key())
-		var newVL VersionList
+		newVL := &VersionList{}
-		for i, version := range vl.Versions {
+		var changed, changedHere bool
-			dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, version.Device, name)
+		for _, fv := range vl.RawVersions {
+			changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.Devices, newVL, ro)
 			if err != nil {
 				return err
 			}
-			_, err := t.Get(dk)
+			changed = changed || changedHere
-			if backend.IsNotFound(err) {
-				continue
-			}
-			if err != nil {
-				return err
-			}
-			newVL.Versions = append(newVL.Versions, version)
 
-			if i == 0 {
+			changedHere, err = checkGlobalsFilterDevices(dk, folder, name, fv.InvalidDevices, newVL, ro)
-				if fi, ok, err := t.getFileTrunc(dk, true); err != nil {
+			if err != nil {
-					return err
+				return err
-				} else if ok {
-					meta.addFile(protocol.GlobalDeviceID, fi)
-				}
 			}
+			changed = changed || changedHere
 		}
 
-		if newLen := len(newVL.Versions); newLen == 0 {
+		if newVL.Empty() {
 			if err := t.Delete(dbi.Key()); err != nil {
 				return err
 			}
-		} else if newLen != len(vl.Versions) {
+		} else if changed {
-			if err := t.Put(dbi.Key(), mustMarshal(&newVL)); err != nil {
+			if err := t.Put(dbi.Key(), mustMarshal(newVL)); err != nil {
 				return err
 			}
 		}
@@ -502,6 +495,30 @@ func (db *Lowlevel) checkGlobals(folder []byte, meta *metadataTracker) error {
 	return t.Commit()
 }
 
+func checkGlobalsFilterDevices(dk, folder, name []byte, devices [][]byte, vl *VersionList, t readOnlyTransaction) (bool, error) {
+	var changed bool
+	var err error
+	for _, device := range devices {
+		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, device, name)
+		if err != nil {
+			return false, err
+		}
+		f, ok, err := t.getFileTrunc(dk, true)
+		if err != nil {
+			return false, err
+		}
+		if !ok {
+			changed = true
+			continue
+		}
+		_, _, _, _, _, _, err = vl.update(folder, device, f, t)
+		if err != nil {
+			return false, err
+		}
+	}
+	return changed, nil
+}
+
 func (db *Lowlevel) getIndexID(device, folder []byte) (protocol.IndexID, error) {
 	key, err := db.keyer.GenerateIndexIDKey(nil, device, folder)
 	if err != nil {
@@ -811,7 +828,7 @@ func (db *Lowlevel) loadMetadataTracker(folder string) *metadataTracker {
 
 func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
 	meta := newMetadataTracker()
-	if err := db.checkGlobals([]byte(folder), meta); err != nil {
+	if err := db.checkGlobals([]byte(folder)); err != nil {
 		return nil, err
 	}
 
@@ -831,8 +848,13 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
 		return nil, err
 	}
 
+	err = t.withGlobal([]byte(folder), nil, true, func(f protocol.FileIntf) bool {
+		meta.addFile(protocol.GlobalDeviceID, f)
+		return true
+	})
+
 	meta.emptyNeeded(protocol.LocalDeviceID)
-	err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f FileIntf) bool {
+	err = t.withNeed([]byte(folder), protocol.LocalDeviceID[:], true, func(f protocol.FileIntf) bool {
 		meta.addNeeded(protocol.LocalDeviceID, f)
 		return true
 	})
@@ -841,7 +863,7 @@ func (db *Lowlevel) recalcMeta(folder string) (*metadataTracker, error) {
 	}
 	for _, device := range meta.devices() {
 		meta.emptyNeeded(device)
-		err = t.withNeed([]byte(folder), device[:], true, func(f FileIntf) bool {
+		err = t.withNeed([]byte(folder), device[:], true, func(f protocol.FileIntf) bool {
 			meta.addNeeded(device, f)
 			return true
 		})
@@ -878,7 +900,7 @@ func (db *Lowlevel) verifyLocalSequence(curSeq int64, folder string) bool {
 		panic(err)
 	}
 	ok := true
-	if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi FileIntf) bool {
+	if err := t.withHaveSequence([]byte(folder), curSeq+1, func(fi protocol.FileIntf) bool {
 		ok = false // we got something, which we should not have
 		return false
 	}); err != nil && !backend.IsClosed(err) {
@@ -1004,6 +1026,6 @@ func (db *Lowlevel) repairSequenceGCLocked(folderStr string, meta *metadataTrack
 // unchanged checks if two files are the same and thus don't need to be updated.
 // Local flags or the invalid bit might change without the version
 // being bumped.
-func unchanged(nf, ef FileIntf) bool {
+func unchanged(nf, ef protocol.FileIntf) bool {
 	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
 }

lib/db/meta.go

@@ -147,7 +147,7 @@ func (m *countsMap) allNeededCounts(dev protocol.DeviceID) Counts {
 
 // addFile adds a file to the counts, adjusting the sequence number as
 // appropriate
-func (m *metadataTracker) addFile(dev protocol.DeviceID, f FileIntf) {
+func (m *metadataTracker) addFile(dev protocol.DeviceID, f protocol.FileIntf) {
 	m.mut.Lock()
 	defer m.mut.Unlock()
 
@@ -186,7 +186,7 @@ func (m *metadataTracker) emptyNeeded(dev protocol.DeviceID) {
 }
 
 // addNeeded adds a file to the needed counts
-func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f FileIntf) {
+func (m *metadataTracker) addNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
 	m.mut.Lock()
 	defer m.mut.Unlock()
 
@@ -201,7 +201,7 @@ func (m *metadataTracker) Sequence(dev protocol.DeviceID) int64 {
 	return m.countsPtr(dev, 0).Sequence
 }
 
-func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) {
+func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f protocol.FileIntf) {
 	if dev == protocol.GlobalDeviceID {
 		return
 	}
@@ -210,7 +210,7 @@ func (m *metadataTracker) updateSeqLocked(dev protocol.DeviceID, f FileIntf) {
 	}
 }
 
-func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
+func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
 	cp := m.countsPtr(dev, flag)
 
 	switch {
@@ -227,7 +227,7 @@ func (m *metadataTracker) addFileLocked(dev protocol.DeviceID, flag uint32, f Fi
 }
 
 // removeFile removes a file from the counts
-func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
+func (m *metadataTracker) removeFile(dev protocol.DeviceID, f protocol.FileIntf) {
 	if f.IsInvalid() && f.FileLocalFlags() == 0 {
 		// This is a remote invalid file; it does not count.
 		return
@@ -250,7 +250,7 @@ func (m *metadataTracker) removeFile(dev protocol.DeviceID, f FileIntf) {
 }
 
 // removeNeeded removes a file from the needed counts
-func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) {
+func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f protocol.FileIntf) {
 	m.mut.Lock()
 	defer m.mut.Unlock()
 
@@ -259,7 +259,7 @@ func (m *metadataTracker) removeNeeded(dev protocol.DeviceID, f FileIntf) {
 	m.removeFileLocked(dev, needFlag, f)
 }
 
-func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f FileIntf) {
+func (m *metadataTracker) removeFileLocked(dev protocol.DeviceID, flag uint32, f protocol.FileIntf) {
 	cp := m.countsPtr(dev, flag)
 
 	switch {

lib/db/schemaupdater.go

@@ -7,7 +7,10 @@
 package db
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/syncthing/syncthing/lib/db/backend"
@@ -23,12 +26,14 @@ import (
 //   7: v0.14.53
 //   8-9: v1.4.0
 //   10-11: v1.6.0
-//   12: v1.7.0
+//   12-13: v1.7.0
 const (
-	dbVersion             = 12
+	dbVersion             = 13
 	dbMinSyncthingVersion = "v1.7.0"
 )
 
+var errFolderMissing = errors.New("folder present in global list but missing in keyer index")
+
 type databaseDowngradeError struct {
 	minSyncthingVersion string
 }
@@ -89,14 +94,14 @@ func (db *schemaUpdater) updateSchema() error {
 		{9, db.updateSchemaTo9},
 		{10, db.updateSchemaTo10},
 		{11, db.updateSchemaTo11},
-		{12, db.updateSchemaTo12},
+		{13, db.updateSchemaTo13},
 	}
 
 	for _, m := range migrations {
 		if prevVersion < m.schemaVersion {
 			l.Infof("Migrating database to schema version %d...", m.schemaVersion)
 			if err := m.migration(int(prevVersion)); err != nil {
-				return err
+				return fmt.Errorf("failed migrating to version %v: %w", m.schemaVersion, err)
 			}
 		}
 	}
@@ -128,8 +133,8 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 	symlinkConv := 0
 	changedFolders := make(map[string]struct{})
 	ignAdded := 0
-	meta := newMetadataTracker() // dummy metadata tracker
+	var gk []byte
-	var gk, buf []byte
+	ro := t.readOnlyTransaction
 
 	for dbi.Next() {
 		folder, ok := db.keyer.FolderFromDeviceFileKey(dbi.Key())
@@ -155,17 +160,27 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if _, ok := changedFolders[string(folder)]; !ok {
 				changedFolders[string(folder)] = struct{}{}
 			}
+			if err := t.Delete(dbi.Key()); err != nil {
+				return err
+			}
 			gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if err != nil {
 				return err
 			}
-			// Purposely pass nil file name to remove from global list,
+			fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
-			// but don't touch meta and needs
+			if backend.IsNotFound(err) {
-			buf, err = t.removeFromGlobal(gk, buf, folder, device, nil, nil)
+				// Shouldn't happen, but not critical.
-			if err != nil && err != errEntryFromGlobalMissing {
+				continue
+			} else if err != nil {
 				return err
 			}
-			if err := t.Delete(dbi.Key()); err != nil {
+			_, _ = fl.pop(device)
+			if len(fl.Versions) == 0 {
+				err = t.Delete(gk)
+			} else {
+				err = t.Put(gk, mustMarshal(&fl))
+			}
+			if err != nil {
 				return err
 			}
 			continue
@@ -199,13 +214,41 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 			if err != nil {
 				return err
 			}
-			if buf, ok, err = t.updateGlobal(gk, buf, folder, device, f, meta); err != nil {
+
+			fl, err := getGlobalVersionsByKeyBefore11(gk, ro)
+			if err != nil && !backend.IsNotFound(err) {
 				return err
-			} else if ok {
+			}
-				if _, ok = changedFolders[string(folder)]; !ok {
+			i := 0
-					changedFolders[string(folder)] = struct{}{}
+			i = sort.Search(len(fl.Versions), func(j int) bool {
+				return fl.Versions[j].Invalid
+			})
+			for ; i < len(fl.Versions); i++ {
+				ordering := fl.Versions[i].Version.Compare(f.Version)
+				shouldInsert := ordering == protocol.Equal
+				if !shouldInsert {
+					shouldInsert, err = shouldInsertBefore(ordering, folder, fl.Versions[i].Device, true, f, ro)
+					if err != nil {
+						return err
+					}
 				}
-				ignAdded++
+				if shouldInsert {
+					nv := FileVersionDeprecated{
+						Device:  device,
+						Version: f.Version,
+						Invalid: true,
+					}
+					fl.insertAt(i, nv)
+					if err := t.Put(gk, mustMarshal(&fl)); err != nil {
+						return err
+					}
+					if _, ok := changedFolders[string(folder)]; !ok {
+						changedFolders[string(folder)] = struct{}{}
+					}
+					ignAdded++
+					break
+				}
+
 			}
 		}
 		if err := t.Checkpoint(); err != nil {
@@ -217,11 +260,6 @@ func (db *schemaUpdater) updateSchema0to1(_ int) error {
 		return err
 	}
 
-	for folder := range changedFolders {
-		if err := db.dropFolderMeta([]byte(folder)); err != nil {
-			return err
-		}
-	}
 	return t.Commit()
 }
 
@@ -239,7 +277,7 @@ func (db *schemaUpdater) updateSchema1to2(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f protocol.FileIntf) bool {
 			sk, putErr = db.keyer.GenerateSequenceKey(sk, folder, f.SequenceNo())
 			if putErr != nil {
 				return false
@@ -274,7 +312,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withGlobal(folder, nil, true, func(f FileIntf) bool {
+		err := withGlobalBefore11(folder, true, func(f protocol.FileIntf) bool {
 			name := []byte(f.FileName())
 			dk, putErr = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name)
 			if putErr != nil {
@@ -289,12 +327,12 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			if ok {
 				v = haveFile.FileVersion()
 			}
-			fv := FileVersion{
+			fv := FileVersionDeprecated{
 				Version: f.FileVersion(),
 				Invalid: f.IsInvalid(),
 				Deleted: f.IsDeleted(),
 			}
-			if !need(fv, ok, v) {
+			if !needDeprecated(fv, ok, v) {
 				return true
 			}
 			nk, putErr = t.keyer.GenerateNeedFileKey(nk, folder, []byte(f.FileName()))
@@ -303,7 +341,7 @@ func (db *schemaUpdater) updateSchema2to3(_ int) error {
 			}
 			putErr = t.Put(nk, nil)
 			return putErr == nil
-		})
+		}, t.readOnlyTransaction)
 		if putErr != nil {
 			return putErr
 		}
@@ -359,7 +397,7 @@ func (db *schemaUpdater) updateSchema5to6(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var iterErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, false, func(f protocol.FileIntf) bool {
 			if !f.IsInvalid() {
 				return true
 			}
@@ -404,7 +442,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var delErr error
-		err := t.withNeedLocal(folder, false, func(f FileIntf) bool {
+		err := withNeedLocalBefore11(folder, false, func(f protocol.FileIntf) bool {
 			name := []byte(f.FileName())
 			gk, delErr = db.keyer.GenerateGlobalVersionKey(gk, folder, name)
 			if delErr != nil {
@@ -421,20 +459,20 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				delErr = t.Delete(key)
 				return delErr == nil
 			}
-			var fl VersionList
+			var fl VersionListDeprecated
 			err = fl.Unmarshal(svl)
 			if err != nil {
 				// This can't happen, but it's ignored everywhere else too,
 				// so lets not act on it.
 				return true
 			}
-			globalFV := FileVersion{
+			globalFV := FileVersionDeprecated{
 				Version: f.FileVersion(),
 				Invalid: f.IsInvalid(),
 				Deleted: f.IsDeleted(),
 			}
 
-			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !need(globalFV, haveLocalFV, localFV.Version) {
+			if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); !needDeprecated(globalFV, haveLocalFV, localFV.Version) {
 				key, err := t.keyer.GenerateNeedFileKey(nk, folder, name)
 				if err != nil {
 					delErr = err
@@ -443,7 +481,7 @@ func (db *schemaUpdater) updateSchema6to7(_ int) error {
 				delErr = t.Delete(key)
 			}
 			return delErr == nil
-		})
+		}, t.readOnlyTransaction)
 		if delErr != nil {
 			return delErr
 		}
@@ -480,6 +518,7 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
 	if err != nil {
 		return err
 	}
+	defer it.Release()
 	for it.Next() {
 		intf, err := t.unmarshalTrunc(it.Value(), false)
 		if backend.IsNotFound(err) {
@@ -510,6 +549,8 @@ func (db *schemaUpdater) rewriteFiles(t readWriteTransaction) error {
 }
 
 func (db *schemaUpdater) updateSchemaTo10(_ int) error {
+	// Rewrites global lists to include a Deleted flag.
+
 	t, err := db.newReadWriteTransaction()
 	if err != nil {
 		return err
@@ -533,7 +574,7 @@ func (db *schemaUpdater) updateSchemaTo10(_ int) error {
 		defer dbi.Release()
 
 		for dbi.Next() {
-			var vl VersionList
+			var vl VersionListDeprecated
 			if err := vl.Unmarshal(dbi.Value()); err != nil {
 				return err
 			}
@@ -592,7 +633,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
 	for _, folderStr := range db.ListFolders() {
 		folder := []byte(folderStr)
 		var putErr error
-		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi FileIntf) bool {
+		err := t.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(fi protocol.FileIntf) bool {
 			f := fi.(FileInfoTruncated)
 			if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() || f.IsInvalid() || f.BlocksHash == nil {
 				return true
@@ -620,7 +661,7 @@ func (db *schemaUpdater) updateSchemaTo11(_ int) error {
 	return t.Commit()
 }
 
-func (db *schemaUpdater) updateSchemaTo12(_ int) error {
+func (db *schemaUpdater) updateSchemaTo13(prev int) error {
 	// Loads and rewrites all files, to deduplicate version vectors.
 
 	t, err := db.newReadWriteTransaction()
@@ -629,9 +670,280 @@ func (db *schemaUpdater) updateSchemaTo12(_ int) error {
 	}
 	defer t.close()
 
-	if err := db.rewriteFiles(t); err != nil {
+	if prev < 12 {
+		if err := db.rewriteFiles(t); err != nil {
+			return err
+		}
+	}
+
+	if err := db.rewriteGlobals(t); err != nil {
 		return err
 	}
 
 	return t.Commit()
 }
+
+func (db *schemaUpdater) rewriteGlobals(t readWriteTransaction) error {
+	it, err := t.NewPrefixIterator([]byte{KeyTypeGlobal})
+	if err != nil {
+		return err
+	}
+	defer it.Release()
+	for it.Next() {
+		var vl VersionListDeprecated
+		if err := vl.Unmarshal(it.Value()); err != nil {
+			// If we crashed during an earlier migration, some version
+			// lists might already be in the new format: Skip those.
+			var nvl VersionList
+			if nerr := nvl.Unmarshal(it.Value()); nerr == nil {
+				continue
+			}
+			return err
+		}
+		if len(vl.Versions) == 0 {
+			if err := t.Delete(it.Key()); err != nil {
+				return err
+			}
+		}
+
+		newVl, err := convertVersionList(vl)
+		if err != nil {
+			return err
+		}
+		if err := t.Put(it.Key(), mustMarshal(&newVl)); err != nil {
+			return err
+		}
+		if err := t.Checkpoint(); err != nil {
+			return err
+		}
+	}
+	it.Release()
+	return it.Error()
+}
+
+func convertVersionList(vl VersionListDeprecated) (VersionList, error) {
+	var newVl VersionList
+	var newPos, oldPos int
+	var lastVersion protocol.Vector
+
+	for _, fv := range vl.Versions {
+		if fv.Invalid {
+			break
+		}
+		oldPos++
+		if lastVersion.Equal(fv.Version) {
+			newVl.RawVersions[newPos].Devices = append(newVl.RawVersions[newPos].Devices, fv.Device)
+			continue
+		}
+		newPos = len(newVl.RawVersions)
+		newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, false, fv.Deleted))
+		lastVersion = fv.Version
+	}
+
+	if oldPos == len(vl.Versions) {
+		return newVl, nil
+	}
+
+	if len(newVl.RawVersions) == 0 {
+		fv := vl.Versions[oldPos]
+		newVl.RawVersions = []FileVersion{newFileVersion(fv.Device, fv.Version, true, fv.Deleted)}
+	}
+	newPos = 0
+outer:
+	for _, fv := range vl.Versions[oldPos:] {
+		for _, nfv := range newVl.RawVersions[newPos:] {
+			switch nfv.Version.Compare(fv.Version) {
+			case protocol.Equal:
+				newVl.RawVersions[newPos].InvalidDevices = append(newVl.RawVersions[newPos].InvalidDevices, fv.Device)
+				lastVersion = fv.Version
+				continue outer
+			case protocol.Lesser:
+				newVl.insertAt(newPos, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
+				lastVersion = fv.Version
+				continue outer
+			case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
+				// The version is invalid, i.e. it looses anyway,
+				// no need to check/get the conflicting file.
+			}
+			newPos++
+		}
+		// Couldn't insert into any existing versions
+		newVl.RawVersions = append(newVl.RawVersions, newFileVersion(fv.Device, fv.Version, true, fv.Deleted))
+		lastVersion = fv.Version
+		newPos++
+	}
+
+	return newVl, nil
+}
+
+func getGlobalVersionsByKeyBefore11(key []byte, t readOnlyTransaction) (VersionListDeprecated, error) {
+	bs, err := t.Get(key)
+	if err != nil {
+		return VersionListDeprecated{}, err
+	}
+
+	var vl VersionListDeprecated
+	if err := vl.Unmarshal(bs); err != nil {
+		return VersionListDeprecated{}, err
+	}
+
+	return vl, nil
+}
+
+func withGlobalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
+	key, err := t.keyer.GenerateGlobalVersionKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key)
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var dk []byte
+	for dbi.Next() {
+		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
+
+		var vl VersionListDeprecated
+		if err := vl.Unmarshal(dbi.Value()); err != nil {
+			return err
+		}
+
+		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
+		if err != nil {
+			return err
+		}
+
+		f, ok, err := t.getFileTrunc(dk, truncate)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+
+		if !fn(f) {
+			return nil
+		}
+	}
+	if err != nil {
+		return err
+	}
+	return dbi.Error()
+}
+
+func withNeedLocalBefore11(folder []byte, truncate bool, fn Iterator, t readOnlyTransaction) error {
+	key, err := t.keyer.GenerateNeedFileKey(nil, folder, nil)
+	if err != nil {
+		return err
+	}
+	dbi, err := t.NewPrefixIterator(key.WithoutName())
+	if err != nil {
+		return err
+	}
+	defer dbi.Release()
+
+	var keyBuf []byte
+	var f protocol.FileIntf
+	var ok bool
+	for dbi.Next() {
+		keyBuf, f, ok, err = getGlobalBefore11(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate, t)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			continue
+		}
+		if !fn(f) {
+			return nil
+		}
+	}
+	return dbi.Error()
+}
+
+func getGlobalBefore11(keyBuf, folder, file []byte, truncate bool, t readOnlyTransaction) ([]byte, protocol.FileIntf, bool, error) {
+	keyBuf, err := t.keyer.GenerateGlobalVersionKey(keyBuf, folder, file)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	bs, err := t.Get(keyBuf)
+	if backend.IsNotFound(err) {
+		return keyBuf, nil, false, nil
+	} else if err != nil {
+		return nil, nil, false, err
+	}
+	var vl VersionListDeprecated
+	if err := vl.Unmarshal(bs); err != nil {
+		return nil, nil, false, err
+	}
+	if len(vl.Versions) == 0 {
+		return nil, nil, false, nil
+	}
+	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	fi, ok, err := t.getFileTrunc(keyBuf, truncate)
+	if err != nil || !ok {
+		return keyBuf, nil, false, err
+	}
+	return keyBuf, fi, true, nil
+}
+
+func (vl *VersionListDeprecated) String() string {
+	var b bytes.Buffer
+	var id protocol.DeviceID
+	b.WriteString("{")
+	for i, v := range vl.Versions {
+		if i > 0 {
+			b.WriteString(", ")
+		}
+		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
+	}
+	b.WriteString("}")
+	return b.String()
+}
+
+func (vl *VersionListDeprecated) pop(device []byte) (FileVersionDeprecated, int) {
+	for i, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
+			return v, i
+		}
+	}
+	return FileVersionDeprecated{}, -1
+}
+
+func (vl *VersionListDeprecated) Get(device []byte) (FileVersionDeprecated, bool) {
+	for _, v := range vl.Versions {
+		if bytes.Equal(v.Device, device) {
+			return v, true
+		}
+	}
+
+	return FileVersionDeprecated{}, false
+}
+
+func (vl *VersionListDeprecated) insertAt(i int, v FileVersionDeprecated) {
+	vl.Versions = append(vl.Versions, FileVersionDeprecated{})
+	copy(vl.Versions[i+1:], vl.Versions[i:])
+	vl.Versions[i] = v
+}
+
+func needDeprecated(global FileVersionDeprecated, haveLocal bool, localVersion protocol.Vector) bool {
+	// We never need an invalid file.
+	if global.Invalid {
+		return false
+	}
+	// We don't need a deleted file if we don't have it.
+	if global.Deleted && !haveLocal {
+		return false
+	}
+	// We don't need the global file if we already have the same version.
+	if haveLocal && localVersion.GreaterEqual(global.Version) {
+		return false
+	}
+	return true
+}

lib/db/set.go

@@ -13,8 +13,6 @@
 package db
 
 import (
-	"time"
-
 	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -31,35 +29,10 @@ type FileSet struct {
 	updateMutex sync.Mutex // protects database updates and the corresponding metadata changes
 }
 
-// FileIntf is the set of methods implemented by both protocol.FileInfo and
-// FileInfoTruncated.
-type FileIntf interface {
-	FileSize() int64
-	FileName() string
-	FileLocalFlags() uint32
-	IsDeleted() bool
-	IsInvalid() bool
-	IsIgnored() bool
-	IsUnsupported() bool
-	MustRescan() bool
-	IsReceiveOnlyChanged() bool
-	IsDirectory() bool
-	IsSymlink() bool
-	ShouldConflict() bool
-	HasPermissionBits() bool
-	SequenceNo() int64
-	BlockSize() int
-	FileVersion() protocol.Vector
-	FileType() protocol.FileInfoType
-	FilePermissions() uint32
-	FileModifiedBy() protocol.ShortID
-	ModTime() time.Time
-}
-
 // The Iterator is called with either a protocol.FileInfo or a
 // FileInfoTruncated (depending on the method) and returns true to
 // continue iteration, false to stop.
-type Iterator func(f FileIntf) bool
+type Iterator func(f protocol.FileIntf) bool
 
 func NewFileSet(folder string, fs fs.Filesystem, db *Lowlevel) *FileSet {
 	return &FileSet{
@@ -335,7 +308,7 @@ func (s *Snapshot) LocalChangedFiles(page, perpage int) []FileInfoTruncated {
 	skip := (page - 1) * perpage
 	get := perpage
 
-	s.WithHaveTruncated(protocol.LocalDeviceID, func(f FileIntf) bool {
+	s.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
 		if !f.IsReceiveOnlyChanged() {
 			return true
 		}
@@ -359,7 +332,7 @@ func (s *Snapshot) RemoteNeedFolderFiles(device protocol.DeviceID, page, perpage
 	files := make([]FileInfoTruncated, 0, perpage)
 	skip := (page - 1) * perpage
 	get := perpage
-	s.WithNeedTruncated(device, func(f FileIntf) bool {
+	s.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
 		if skip > 0 {
 			skip--
 			return true
@@ -497,7 +470,7 @@ func normalizeFilenamesAndDropDuplicates(fs []protocol.FileInfo) []protocol.File
 }
 
 func nativeFileIterator(fn Iterator) Iterator {
-	return func(fi FileIntf) bool {
+	return func(fi protocol.FileIntf) bool {
 		switch f := fi.(type) {
 		case protocol.FileInfo:
 			f.Name = osutil.NativeFilename(f.Name)

lib/db/set_test.go

@@ -48,7 +48,7 @@ func globalList(s *db.FileSet) []protocol.FileInfo {
 	var fs []protocol.FileInfo
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithGlobal(func(fi db.FileIntf) bool {
+	snap.WithGlobal(func(fi protocol.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -59,7 +59,7 @@ func globalListPrefixed(s *db.FileSet, prefix string) []db.FileInfoTruncated {
 	var fs []db.FileInfoTruncated
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
+	snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 		fs = append(fs, f)
 		return true
@@ -71,7 +71,7 @@ func haveList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithHave(n, func(fi db.FileIntf) bool {
+	snap.WithHave(n, func(fi protocol.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -83,7 +83,7 @@ func haveListPrefixed(s *db.FileSet, n protocol.DeviceID, prefix string) []db.Fi
 	var fs []db.FileInfoTruncated
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithPrefixedHaveTruncated(n, prefix, func(fi db.FileIntf) bool {
+	snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 		fs = append(fs, f)
 		return true
@@ -95,7 +95,7 @@ func needList(s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
 	var fs []protocol.FileInfo
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithNeed(n, func(fi db.FileIntf) bool {
+	snap.WithNeed(n, func(fi protocol.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		fs = append(fs, f)
 		return true
@@ -998,7 +998,7 @@ func TestWithHaveSequence(t *testing.T) {
 	i := 2
 	snap := s.Snapshot()
 	defer snap.Release()
-	snap.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
+	snap.WithHaveSequence(int64(i), func(fi protocol.FileIntf) bool {
 		if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
 			t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
 		}
@@ -1049,7 +1049,7 @@ loop:
 		default:
 		}
 		snap := s.Snapshot()
-		snap.WithHaveSequence(prevSeq+1, func(fi db.FileIntf) bool {
+		snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileIntf) bool {
 			if fi.SequenceNo() < prevSeq+1 {
 				t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
 			}
@@ -1527,8 +1527,8 @@ func TestSequenceIndex(t *testing.T) {
 
 	// Start a routine to walk the sequence index and inspect the result.
 
-	seen := make(map[string]db.FileIntf)
+	seen := make(map[string]protocol.FileIntf)
-	latest := make([]db.FileIntf, 0, len(local))
+	latest := make([]protocol.FileIntf, 0, len(local))
 	var seq int64
 	t0 := time.Now()
 
@@ -1539,7 +1539,7 @@ func TestSequenceIndex(t *testing.T) {
 		// update has happened since our last iteration.
 		latest = latest[:0]
 		snap := s.Snapshot()
-		snap.WithHaveSequence(seq+1, func(f db.FileIntf) bool {
+		snap.WithHaveSequence(seq+1, func(f protocol.FileIntf) bool {
 			seen[f.FileName()] = f
 			latest = append(latest, f)
 			seq = f.SequenceNo()
@@ -1644,7 +1644,7 @@ func TestUpdateWithOneFileTwice(t *testing.T) {
 	snap := s.Snapshot()
 	defer snap.Release()
 	count := 0
-	snap.WithHaveSequence(0, func(f db.FileIntf) bool {
+	snap.WithHaveSequence(0, func(f protocol.FileIntf) bool {
 		count++
 		return true
 	})

lib/db/structs.go

@@ -12,7 +12,6 @@ package db
 import (
 	"bytes"
 	"fmt"
-	"sort"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -196,98 +195,317 @@ func (vl VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
 	b.WriteString("{")
-	for i, v := range vl.Versions {
+	for i, v := range vl.RawVersions {
 		if i > 0 {
 			b.WriteString(", ")
 		}
-		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%v, {", v.Version)
-		fmt.Fprintf(&b, "{%v, %v}", v.Version, id)
+		for j, dev := range v.Devices {
+			if j > 0 {
+				b.WriteString(", ")
+			}
+			copy(id[:], dev)
+			fmt.Fprint(&b, id.Short())
+		}
+		b.WriteString("}, {")
+		for j, dev := range v.InvalidDevices {
+			if j > 0 {
+				b.WriteString(", ")
+			}
+			copy(id[:], dev)
+			fmt.Fprint(&b, id.Short())
+		}
+		fmt.Fprint(&b, "}}")
 	}
 	b.WriteString("}")
 	return b.String()
 }
 
 // update brings the VersionList up to date with file. It returns the updated
-// VersionList, a potentially removed old FileVersion and its index, as well as
+// VersionList, a device that has the global/newest version, a device that previously
-// the index where the new FileVersion was inserted.
+// had the global/newest version, a boolean indicating if the global version has
-func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int, err error) {
+// changed and if any error occurred (only possible in db interaction).
-	vl, removedFV, removedAt = vl.pop(device)
+func (vl *VersionList) update(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (FileVersion, FileVersion, FileVersion, bool, bool, bool, error) {
-
+	if len(vl.RawVersions) == 0 {
-	nv := FileVersion{
+		nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
-		Device:  device,
+		vl.RawVersions = append(vl.RawVersions, nv)
-		Version: file.Version,
+		return nv, FileVersion{}, FileVersion{}, false, false, true, nil
-		Invalid: file.IsInvalid(),
-		Deleted: file.IsDeleted(),
-	}
-	i := 0
-	if nv.Invalid {
-		i = sort.Search(len(vl.Versions), func(j int) bool {
-			return vl.Versions[j].Invalid
-		})
-	}
-	for ; i < len(vl.Versions); i++ {
-		switch vl.Versions[i].Version.Compare(file.Version) {
-		case protocol.Equal:
-			fallthrough
-
-		case protocol.Lesser:
-			// The version at this point in the list is equal to or lesser
-			// ("older") than us. We insert ourselves in front of it.
-			vl = vl.insertAt(i, nv)
-			return vl, removedFV, removedAt, i, nil
-
-		case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
-			// The version at this point is in conflict with us. We must pull
-			// the actual file metadata to determine who wins. If we win, we
-			// insert ourselves in front of the loser here. (The "Lesser" and
-			// "Greater" in the condition above is just based on the device
-			// IDs in the version vector, which is not the only thing we use
-			// to determine the winner.)
-			//
-			// A surprise missing file entry here is counted as a win for us.
-			if of, ok, err := t.getFile(folder, vl.Versions[i].Device, []byte(file.Name)); err != nil {
-				return vl, removedFV, removedAt, i, err
-			} else if !ok || file.WinsConflict(of) {
-				vl = vl.insertAt(i, nv)
-				return vl, removedFV, removedAt, i, nil
-			}
-		}
 	}
 
-	// We didn't find a position for an insert above, so append to the end.
+	// Get the current global (before updating)
-	vl.Versions = append(vl.Versions, nv)
+	oldFV, haveOldGlobal := vl.GetGlobal()
 
-	return vl, removedFV, removedAt, len(vl.Versions) - 1, nil
+	// Remove ourselves first
+	removedFV, haveRemoved, _, err := vl.pop(folder, device, []byte(file.FileName()), t)
+	if err == nil {
+		// Find position and insert the file
+		err = vl.insert(folder, device, file, t)
+	}
+	if err != nil {
+		return FileVersion{}, FileVersion{}, FileVersion{}, false, false, false, err
+	}
+
+	newFV, _ := vl.GetGlobal() // We just inserted something above, can't be empty
+
+	if !haveOldGlobal {
+		return newFV, FileVersion{}, removedFV, false, haveRemoved, true, nil
+	}
+
+	globalChanged := true
+	if oldFV.IsInvalid() == newFV.IsInvalid() && oldFV.Version.Equal(newFV.Version) {
+		globalChanged = false
+	}
+
+	return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
 }
 
-func (vl VersionList) insertAt(i int, v FileVersion) VersionList {
+func (vl *VersionList) insert(folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) error {
-	vl.Versions = append(vl.Versions, FileVersion{})
+	var added bool
-	copy(vl.Versions[i+1:], vl.Versions[i:])
+	var err error
-	vl.Versions[i] = v
+	i := 0
-	return vl
+	for ; i < len(vl.RawVersions); i++ {
+		// Insert our new version
+		added, err = vl.checkInsertAt(i, folder, device, file, t)
+		if err != nil {
+			return err
+		}
+		if added {
+			break
+		}
+	}
+	if i == len(vl.RawVersions) {
+		// Append to the end
+		vl.RawVersions = append(vl.RawVersions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
+	}
+	return nil
+}
+
+func (vl *VersionList) insertAt(i int, v FileVersion) {
+	vl.RawVersions = append(vl.RawVersions, FileVersion{})
+	copy(vl.RawVersions[i+1:], vl.RawVersions[i:])
+	vl.RawVersions[i] = v
 }
 
 // pop returns the VersionList without the entry for the given device, as well
-// as the removed FileVersion and the position, where that FileVersion was.
+// as the removed FileVersion, whether it was found/removed at all and whether
-// If there is no FileVersion for the given device, the position is -1.
+// the global changed in the process.
-func (vl VersionList) pop(device []byte) (VersionList, FileVersion, int) {
+func (vl *VersionList) pop(folder, device, name []byte, t readOnlyTransaction) (FileVersion, bool, bool, error) {
-	for i, v := range vl.Versions {
+	invDevice, i, j, ok := vl.findDevice(device)
-		if bytes.Equal(v.Device, device) {
+	if !ok {
-			vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
+		return FileVersion{}, false, false, nil
-			return vl, v, i
-		}
 	}
-	return vl, FileVersion{}, -1
+	globalPos := vl.findGlobal()
+
+	if vl.RawVersions[i].deviceCount() == 1 {
+		fv := vl.RawVersions[i]
+		vl.popVersionAt(i)
+		return fv, true, globalPos == i, nil
+	}
+
+	if invDevice {
+		vl.RawVersions[i].InvalidDevices = popDeviceAt(vl.RawVersions[i].InvalidDevices, j)
+	} else {
+		vl.RawVersions[i].Devices = popDeviceAt(vl.RawVersions[i].Devices, j)
+	}
+	// If the last valid device of the previous global was removed above,
+	// the next entry is now the global entry (unless all entries are invalid).
+	if len(vl.RawVersions[i].Devices) == 0 && globalPos == i {
+		return vl.RawVersions[i], true, globalPos == vl.findGlobal(), nil
+	}
+	return vl.RawVersions[i], true, false, nil
 }
 
-func (vl VersionList) Get(device []byte) (FileVersion, bool) {
+// Get returns a FileVersion that contains the given device and whether it has
-	for _, v := range vl.Versions {
+// been found at all.
-		if bytes.Equal(v.Device, device) {
+func (vl *VersionList) Get(device []byte) (FileVersion, bool) {
-			return v, true
+	_, i, _, ok := vl.findDevice(device)
+	if !ok {
+		return FileVersion{}, false
+	}
+	return vl.RawVersions[i], true
+}
+
+// GetGlobal returns the current global FileVersion. The returned FileVersion
+// may be invalid, if all FileVersions are invalid. Returns false only if
+// VersionList is empty.
+func (vl *VersionList) GetGlobal() (FileVersion, bool) {
+	i := vl.findGlobal()
+	if i == -1 {
+		return FileVersion{}, false
+	}
+	return vl.RawVersions[i], true
+}
+
+func (vl *VersionList) Empty() bool {
+	return len(vl.RawVersions) == 0
+}
+
+// findGlobal returns the first version that isn't invalid, or if all versions are
+// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
+func (vl *VersionList) findGlobal() int {
+	for i, fv := range vl.RawVersions {
+		if !fv.IsInvalid() {
+			return i
 		}
 	}
+	if len(vl.RawVersions) == 0 {
+		return -1
+	}
+	return 0
+}
 
-	return FileVersion{}, false
+// findDevices returns whether the device is in InvalidVersions or Versions and
+// in InvalidDevices or Devices (true for invalid), the positions in the version
+// and device slices and whether it has been found at all.
+func (vl *VersionList) findDevice(device []byte) (bool, int, int, bool) {
+	for i, v := range vl.RawVersions {
+		if j := deviceIndex(v.Devices, device); j != -1 {
+			return false, i, j, true
+		}
+		if j := deviceIndex(v.InvalidDevices, device); j != -1 {
+			return true, i, j, true
+		}
+	}
+	return false, -1, -1, false
+}
+
+func (vl *VersionList) popVersion(version protocol.Vector) (FileVersion, bool) {
+	i := vl.versionIndex(version)
+	if i == -1 {
+		return FileVersion{}, false
+	}
+	fv := vl.RawVersions[i]
+	vl.popVersionAt(i)
+	return fv, true
+}
+
+func (vl *VersionList) versionIndex(version protocol.Vector) int {
+	for i, v := range vl.RawVersions {
+		if version.Equal(v.Version) {
+			return i
+		}
+	}
+	return -1
+}
+
+func (vl *VersionList) popVersionAt(i int) {
+	vl.RawVersions = append(vl.RawVersions[:i], vl.RawVersions[i+1:]...)
+}
+
+// checkInsertAt determines if the given device and associated file should be
+// inserted into the FileVersion at position i or into a new FileVersion at
+// position i.
+func (vl *VersionList) checkInsertAt(i int, folder, device []byte, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
+	ordering := vl.RawVersions[i].Version.Compare(file.FileVersion())
+	if ordering == protocol.Equal {
+		if !file.IsInvalid() {
+			vl.RawVersions[i].Devices = append(vl.RawVersions[i].Devices, device)
+		} else {
+			vl.RawVersions[i].InvalidDevices = append(vl.RawVersions[i].InvalidDevices, device)
+		}
+		return true, nil
+	}
+	existingDevice, _ := vl.RawVersions[i].FirstDevice()
+	insert, err := shouldInsertBefore(ordering, folder, existingDevice, vl.RawVersions[i].IsInvalid(), file, t)
+	if err != nil {
+		return false, err
+	}
+	if insert {
+		vl.insertAt(i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
+		return true, nil
+	}
+	return false, nil
+}
+
+// shouldInsertBefore determines whether the file comes before an existing
+// entry, given the version ordering (existing compared to new one), existing
+// device and if the existing version is invalid.
+func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileIntf, t readOnlyTransaction) (bool, error) {
+	switch ordering {
+	case protocol.Lesser:
+		// The version at this point in the list is lesser
+		// ("older") than us. We insert ourselves in front of it.
+		return true, nil
+
+	case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
+		// The version in conflict with us.
+		// Check if we can shortcut due to one being invalid.
+		if existingInvalid != file.IsInvalid() {
+			return existingInvalid, nil
+		}
+		// We must pull the actual file metadata to determine who wins.
+		// If we win, we insert ourselves in front of the loser here.
+		// (The "Lesser" and "Greater" in the condition above is just
+		// based on the device IDs in the version vector, which is not
+		// the only thing we use to determine the winner.)
+		of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
+		if err != nil {
+			return false, err
+		}
+		// A surprise missing file entry here is counted as a win for us.
+		if !ok {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		if protocol.WinsConflict(file, of) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func deviceIndex(devices [][]byte, device []byte) int {
+	for i, dev := range devices {
+		if bytes.Equal(device, dev) {
+			return i
+		}
+	}
+	return -1
+}
+
+func popDeviceAt(devices [][]byte, i int) [][]byte {
+	return append(devices[:i], devices[i+1:]...)
+}
+
+func popDevice(devices [][]byte, device []byte) ([][]byte, bool) {
+	i := deviceIndex(devices, device)
+	if i == -1 {
+		return devices, false
+	}
+	return popDeviceAt(devices, i), true
+}
+
+func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) FileVersion {
+	fv := FileVersion{
+		Version: version,
+		Deleted: deleted,
+	}
+	if invalid {
+		fv.InvalidDevices = [][]byte{device}
+	} else {
+		fv.Devices = [][]byte{device}
+	}
+	return fv
+}
+
+func (fv FileVersion) FirstDevice() ([]byte, bool) {
+	if len(fv.Devices) != 0 {
+		return fv.Devices[0], true
+	}
+	if len(fv.InvalidDevices) != 0 {
+		return fv.InvalidDevices[0], true
+	}
+	return nil, false
+}
+
+func (fv FileVersion) IsInvalid() bool {
+	return len(fv.Devices) == 0
+}
+
+func (fv FileVersion) deviceCount() int {
+	return len(fv.Devices) + len(fv.InvalidDevices)
 }
 
 type fileList []protocol.FileInfo

lib/db/structs.pb.go

@@ -26,10 +26,10 @@ var _ = math.Inf
 const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type FileVersion struct {
-	Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
+	Version        protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
-	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
+	Deleted        bool            `protobuf:"varint,2,opt,name=deleted,proto3" json:"deleted,omitempty"`
-	Invalid bool            `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	Devices        [][]byte        `protobuf:"bytes,3,rep,name=devices,proto3" json:"devices,omitempty"`
-	Deleted bool            `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
+	InvalidDevices [][]byte        `protobuf:"bytes,4,rep,name=invalid_devices,json=invalidDevices,proto3" json:"invalid_devices,omitempty"`
 }
 
 func (m *FileVersion) Reset()         { *m = FileVersion{} }
@@ -66,7 +66,7 @@ func (m *FileVersion) XXX_DiscardUnknown() {
 var xxx_messageInfo_FileVersion proto.InternalMessageInfo
 
 type VersionList struct {
-	Versions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
+	RawVersions []FileVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
 }
 
 func (m *VersionList) Reset()      { *m = VersionList{} }
@@ -318,6 +318,82 @@ func (m *CountsSet) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_CountsSet proto.InternalMessageInfo
 
+type FileVersionDeprecated struct {
+	Version protocol.Vector `protobuf:"bytes,1,opt,name=version,proto3" json:"version"`
+	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
+	Invalid bool            `protobuf:"varint,3,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	Deleted bool            `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"`
+}
+
+func (m *FileVersionDeprecated) Reset()         { *m = FileVersionDeprecated{} }
+func (m *FileVersionDeprecated) String() string { return proto.CompactTextString(m) }
+func (*FileVersionDeprecated) ProtoMessage()    {}
+func (*FileVersionDeprecated) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e774e8f5f348d14d, []int{7}
+}
+func (m *FileVersionDeprecated) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *FileVersionDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_FileVersionDeprecated.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *FileVersionDeprecated) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_FileVersionDeprecated.Merge(m, src)
+}
+func (m *FileVersionDeprecated) XXX_Size() int {
+	return m.ProtoSize()
+}
+func (m *FileVersionDeprecated) XXX_DiscardUnknown() {
+	xxx_messageInfo_FileVersionDeprecated.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_FileVersionDeprecated proto.InternalMessageInfo
+
+type VersionListDeprecated struct {
+	Versions []FileVersionDeprecated `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions"`
+}
+
+func (m *VersionListDeprecated) Reset()      { *m = VersionListDeprecated{} }
+func (*VersionListDeprecated) ProtoMessage() {}
+func (*VersionListDeprecated) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e774e8f5f348d14d, []int{8}
+}
+func (m *VersionListDeprecated) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *VersionListDeprecated) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_VersionListDeprecated.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *VersionListDeprecated) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VersionListDeprecated.Merge(m, src)
+}
+func (m *VersionListDeprecated) XXX_Size() int {
+	return m.ProtoSize()
+}
+func (m *VersionListDeprecated) XXX_DiscardUnknown() {
+	xxx_messageInfo_VersionListDeprecated.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VersionListDeprecated proto.InternalMessageInfo
+
 func init() {
 	proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
 	proto.RegisterType((*VersionList)(nil), "db.VersionList")
@@ -326,61 +402,68 @@ func init() {
 	proto.RegisterType((*IndirectionHashesOnly)(nil), "db.IndirectionHashesOnly")
 	proto.RegisterType((*Counts)(nil), "db.Counts")
 	proto.RegisterType((*CountsSet)(nil), "db.CountsSet")
+	proto.RegisterType((*FileVersionDeprecated)(nil), "db.FileVersionDeprecated")
+	proto.RegisterType((*VersionListDeprecated)(nil), "db.VersionListDeprecated")
 }
 
 func init() { proto.RegisterFile("structs.proto", fileDescriptor_e774e8f5f348d14d) }
 
 var fileDescriptor_e774e8f5f348d14d = []byte{
-	// 774 bytes of a gzipped FileDescriptorProto
+	// 861 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x8f, 0xe3, 0x44,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x4f, 0x8f, 0xdb, 0x54,
-	0x10, 0x8d, 0x37, 0x71, 0x3e, 0xca, 0x49, 0xd8, 0x6d, 0x96, 0x91, 0x15, 0x09, 0xc7, 0x0a, 0x5a,
+	0x10, 0x8f, 0x9b, 0xff, 0xe3, 0x64, 0xdb, 0xbe, 0x76, 0x57, 0x66, 0x25, 0x1c, 0xcb, 0x08, 0x61,
-	0xc9, 0xe2, 0x90, 0xc0, 0xec, 0x0d, 0x24, 0x0e, 0x61, 0x35, 0x22, 0x12, 0x62, 0x51, 0x67, 0xb5,
+	0x71, 0x48, 0x60, 0x7b, 0xa3, 0x12, 0x42, 0x61, 0x55, 0x11, 0x09, 0x51, 0xf4, 0xb6, 0xf4, 0x80,
-	0xa7, 0x95, 0x22, 0x7f, 0x74, 0x92, 0xd6, 0x38, 0xee, 0xe0, 0xee, 0xcc, 0xc8, 0xf3, 0x17, 0xb8,
+	0x2a, 0x45, 0xb6, 0xf3, 0x92, 0x3c, 0xd5, 0xf1, 0x0b, 0x7e, 0xce, 0xae, 0xdc, 0x4f, 0xc1, 0x05,
-	0x70, 0xe4, 0x38, 0x17, 0xfe, 0xcb, 0x1c, 0xe7, 0x88, 0x38, 0x44, 0x90, 0x70, 0x80, 0x7f, 0x81,
+	0x89, 0x03, 0x87, 0x5e, 0xf8, 0x2e, 0x7b, 0xec, 0x11, 0x71, 0x88, 0x20, 0xcb, 0x01, 0xbe, 0x05,
-	0xba, 0xdb, 0x76, 0x3c, 0x73, 0x61, 0x6e, 0x55, 0xaf, 0x2a, 0xa9, 0xaa, 0xf7, 0x9e, 0x1b, 0x7a,
+	0x7a, 0xf3, 0x6c, 0xc7, 0x1b, 0x0e, 0xb4, 0xb7, 0x99, 0xdf, 0xcc, 0xf3, 0xcc, 0xfc, 0xe6, 0xe7,
-	0x5c, 0xa4, 0xbb, 0x50, 0xf0, 0xf1, 0x36, 0x65, 0x82, 0xa1, 0x67, 0x51, 0x30, 0xf8, 0x2c, 0x25,
+	0x81, 0xbe, 0x4c, 0x93, 0x4d, 0x98, 0xca, 0xe1, 0x3a, 0x11, 0xa9, 0x20, 0x77, 0x66, 0xc1, 0xe9,
-	0x5b, 0xc6, 0x27, 0x0a, 0x08, 0x76, 0xcb, 0xc9, 0x8a, 0xad, 0x98, 0x4a, 0x54, 0xa4, 0x1b, 0x07,
+	0x07, 0x09, 0x5b, 0x0b, 0x39, 0x42, 0x20, 0xd8, 0xcc, 0x47, 0x0b, 0xb1, 0x10, 0xe8, 0xa0, 0xa5,
-	0x67, 0x31, 0x0d, 0x74, 0x4b, 0xc8, 0xe2, 0x49, 0x40, 0xb6, 0x1a, 0x1f, 0xfd, 0x6c, 0x80, 0x75,
+	0x13, 0x4f, 0x4f, 0x22, 0x1e, 0xe8, 0x94, 0x50, 0x44, 0xa3, 0x80, 0xad, 0x35, 0xee, 0xfe, 0x62,
-	0x41, 0x63, 0xf2, 0x9e, 0xa4, 0x9c, 0xb2, 0x04, 0x7d, 0x01, 0xad, 0x2b, 0x1d, 0xda, 0x86, 0x6b,
+	0x80, 0xf9, 0x84, 0x47, 0xec, 0x39, 0x4b, 0x24, 0x17, 0x31, 0xf9, 0x04, 0xda, 0x97, 0xda, 0xb4,
-	0x78, 0xd6, 0xf9, 0xf3, 0x71, 0xf1, 0xab, 0xf1, 0x7b, 0x12, 0x0a, 0x96, 0x4e, 0x1b, 0x77, 0xfb,
+	0x0c, 0xc7, 0xf0, 0xcc, 0xb3, 0x7b, 0xc3, 0xe2, 0xd5, 0xf0, 0x39, 0x0b, 0x53, 0x91, 0x8c, 0x1b,
-	0x61, 0x0d, 0x17, 0x6d, 0xe8, 0x0c, 0x9a, 0x11, 0xb9, 0xa2, 0x21, 0xb1, 0x9f, 0xb9, 0x86, 0xd7,
+	0xd7, 0xdb, 0x41, 0x8d, 0x16, 0x69, 0xc4, 0x82, 0xf6, 0x8c, 0x45, 0x2c, 0x65, 0x33, 0xeb, 0x8e,
-	0xc5, 0x79, 0x86, 0x6c, 0x68, 0xd1, 0xe4, 0xca, 0x8f, 0x69, 0x64, 0xd7, 0x5d, 0xc3, 0x6b, 0xe3,
+	0x63, 0x78, 0x1d, 0x5a, 0xb8, 0x3a, 0x72, 0xc9, 0x43, 0x26, 0xad, 0xba, 0x53, 0xf7, 0x7a, 0xb4,
-	0x22, 0x95, 0x95, 0x88, 0xc4, 0x44, 0x90, 0xc8, 0x6e, 0xe8, 0x4a, 0x9e, 0x8e, 0x2e, 0xc0, 0xca,
+	0x70, 0xc9, 0x47, 0x70, 0x97, 0xc7, 0x97, 0x7e, 0xc4, 0x67, 0xd3, 0x22, 0xa3, 0x81, 0x19, 0x47,
-	0x17, 0xf9, 0x9e, 0x72, 0x81, 0xbe, 0x84, 0x76, 0x3e, 0x85, 0xdb, 0x86, 0x5b, 0xf7, 0xac, 0xf3,
+	0x39, 0x7c, 0xae, 0x51, 0xf7, 0x3b, 0x30, 0xf3, 0xce, 0xbe, 0xe6, 0x32, 0x25, 0x5f, 0x40, 0x27,
-	0x8f, 0xc6, 0x51, 0x30, 0xae, 0xec, 0x9b, 0x2f, 0x53, 0xb6, 0x7d, 0xd5, 0xf8, 0xf5, 0x76, 0x58,
+	0x2f, 0x2b, 0x2d, 0xc3, 0xa9, 0x7b, 0xe6, 0xd9, 0xdd, 0xe1, 0x2c, 0x18, 0x56, 0x06, 0x18, 0x3f,
-	0x1b, 0xfd, 0x66, 0xc2, 0x0b, 0xd9, 0x35, 0x4b, 0x96, 0xec, 0x5d, 0xba, 0x4b, 0x42, 0x5f, 0x90,
+	0x50, 0xdd, 0xed, 0xb6, 0x03, 0x93, 0xfa, 0x57, 0x39, 0x26, 0x69, 0xf9, 0xea, 0xb3, 0xc6, 0xcf,
-	0x08, 0x21, 0x68, 0x24, 0xfe, 0x86, 0xa8, 0xc3, 0x3a, 0x58, 0xc5, 0x12, 0xe3, 0xf4, 0x86, 0xa8,
+	0xaf, 0x07, 0x35, 0xf7, 0xd7, 0x26, 0xdc, 0x57, 0x8f, 0x26, 0xf1, 0x5c, 0x3c, 0x4b, 0x36, 0x71,
-	0x15, 0xeb, 0x58, 0xc5, 0xe8, 0x53, 0x80, 0x0d, 0x8b, 0xe8, 0x92, 0x92, 0x68, 0xc1, 0x6d, 0x53,
+	0xe8, 0xab, 0x7e, 0x09, 0x34, 0x62, 0x7f, 0xc5, 0x70, 0xf0, 0x2e, 0x45, 0x5b, 0x61, 0x92, 0xbf,
-	0x55, 0x3a, 0x05, 0x32, 0x47, 0x1f, 0xc0, 0x2a, 0xcb, 0x41, 0x66, 0x77, 0x5d, 0xc3, 0x6b, 0x4c,
+	0x62, 0x56, 0xdd, 0x31, 0xbc, 0x3a, 0x45, 0x9b, 0xbc, 0x0f, 0xb0, 0x12, 0x33, 0x3e, 0xe7, 0x6c,
-	0xbf, 0x96, 0x7b, 0xfc, 0xb1, 0x1f, 0xbe, 0x5e, 0x51, 0xb1, 0xde, 0x05, 0xe3, 0x90, 0x6d, 0x26,
+	0x36, 0x95, 0x56, 0x13, 0x23, 0xdd, 0x02, 0xb9, 0x20, 0x2f, 0xc0, 0x2c, 0xc3, 0x41, 0x66, 0xf5,
-	0x3c, 0x4b, 0x42, 0xb1, 0xa6, 0xc9, 0xaa, 0x12, 0x55, 0x65, 0x18, 0xcf, 0xd7, 0x2c, 0x15, 0xb3,
+	0x1c, 0xc3, 0x6b, 0x8c, 0x1f, 0xab, 0xb6, 0x7e, 0xdf, 0x0e, 0x1e, 0x2d, 0x78, 0xba, 0xdc, 0x04,
-	0x37, 0xb8, 0x1c, 0x37, 0xcd, 0xaa, 0x02, 0x74, 0x9e, 0x26, 0xc0, 0x00, 0xda, 0x9c, 0xfc, 0xb4,
+	0xc3, 0x50, 0xac, 0x46, 0x32, 0x8b, 0xc3, 0x74, 0xc9, 0xe3, 0x45, 0xc5, 0xaa, 0xae, 0x69, 0x78,
-	0x23, 0x49, 0x48, 0x6c, 0x50, 0xcb, 0x96, 0x39, 0x7a, 0x05, 0x7d, 0x9e, 0x6d, 0x62, 0x9a, 0x5c,
+	0xb1, 0x14, 0x49, 0x3a, 0x39, 0xa7, 0x65, 0xb9, 0x71, 0x56, 0x5d, 0x50, 0xf7, 0xed, 0x16, 0x74,
-	0x2e, 0x84, 0x9f, 0xae, 0x88, 0xb0, 0x5f, 0xa8, 0xe3, 0x7b, 0x39, 0xfa, 0x4e, 0x81, 0x68, 0x08,
+	0x0a, 0x1d, 0xc9, 0x7e, 0xd8, 0xb0, 0x38, 0x64, 0x16, 0x60, 0xb3, 0xa5, 0x4f, 0x3e, 0x84, 0x23,
-	0x56, 0x10, 0xb3, 0xf0, 0x92, 0x2f, 0xd6, 0x3e, 0x5f, 0xdb, 0x48, 0x09, 0x09, 0x1a, 0xfa, 0xce,
+	0x99, 0xad, 0x22, 0x1e, 0xbf, 0x9c, 0xa6, 0x7e, 0xb2, 0x60, 0xa9, 0x75, 0x1f, 0x87, 0xef, 0xe7,
-	0xe7, 0x6b, 0xf4, 0x39, 0x34, 0x44, 0xb6, 0xd5, 0x12, 0xf7, 0xcf, 0xcf, 0x4e, 0x2b, 0x95, 0x2c,
+	0xe8, 0x33, 0x04, 0xc9, 0x00, 0xcc, 0x20, 0x12, 0xe1, 0x4b, 0x39, 0x5d, 0xfa, 0x72, 0x69, 0x11,
-	0x67, 0x5b, 0x82, 0x55, 0x0f, 0x72, 0xc1, 0xda, 0x92, 0x74, 0x43, 0xb9, 0x16, 0x4e, 0x4a, 0xdc,
+	0xc7, 0xf0, 0x7a, 0x14, 0x34, 0xf4, 0x95, 0x2f, 0x97, 0xe4, 0x63, 0x68, 0xa4, 0xd9, 0x9a, 0xa1,
-	0xc3, 0x55, 0x48, 0x8e, 0x2b, 0x19, 0x4c, 0xb8, 0x6d, 0xb9, 0x86, 0x67, 0x9e, 0x48, 0xf8, 0x81,
+	0x02, 0x8e, 0xce, 0x4e, 0xf6, 0x2d, 0x95, 0x2c, 0x67, 0x6b, 0x46, 0x31, 0x87, 0x38, 0x60, 0xae,
-	0xa3, 0x09, 0xe8, 0xe1, 0x0b, 0xa5, 0x4d, 0x4f, 0xd6, 0xa7, 0xcf, 0x0f, 0xfb, 0x61, 0x17, 0xfb,
+	0x59, 0xb2, 0xe2, 0x52, 0xef, 0xb1, 0xe1, 0x18, 0x5e, 0x9f, 0x56, 0x21, 0x55, 0xae, 0x64, 0x30,
-	0xd7, 0x53, 0x59, 0x98, 0xd3, 0x1b, 0x82, 0x3b, 0x41, 0x11, 0xca, 0x99, 0x31, 0x0b, 0xfd, 0x78,
+	0x96, 0x96, 0xe9, 0x18, 0x5e, 0x73, 0x4f, 0xc2, 0x37, 0x92, 0x8c, 0x40, 0x17, 0x9f, 0xe2, 0x6e,
-	0xb1, 0x8c, 0xfd, 0x15, 0xb7, 0xff, 0x69, 0xa9, 0xa1, 0xa0, 0xb0, 0x0b, 0x09, 0xa1, 0x11, 0x74,
+	0xfa, 0x2a, 0x3e, 0xbe, 0xb7, 0xdb, 0x0e, 0x7a, 0xd4, 0xbf, 0x1a, 0xab, 0xc0, 0x05, 0x7f, 0xc5,
-	0x73, 0xc2, 0xf4, 0x8d, 0xff, 0xb6, 0xd4, 0x91, 0x56, 0x0e, 0xaa, 0x2b, 0x2b, 0xc6, 0x6c, 0x3e,
+	0x68, 0x37, 0x28, 0x4c, 0x55, 0x33, 0x12, 0xa1, 0x1f, 0x4d, 0xe7, 0x91, 0xbf, 0x90, 0xd6, 0xdf,
-	0x30, 0x26, 0xf2, 0x4e, 0x66, 0x96, 0xbf, 0x6b, 0x4f, 0xfb, 0x87, 0xfd, 0x10, 0xb0, 0x7f, 0x3d,
+	0x6d, 0x2c, 0x0a, 0x88, 0x3d, 0x51, 0x10, 0x71, 0xa1, 0x97, 0x13, 0xa6, 0x67, 0xfc, 0xa7, 0x8d,
-	0xd3, 0xe8, 0xc9, 0xdc, 0xaf, 0xa0, 0x9f, 0xb0, 0x45, 0x95, 0x80, 0xb6, 0xfa, 0xab, 0x5e, 0xc2,
+	0x43, 0x9a, 0x39, 0x88, 0x53, 0x56, 0xa4, 0xde, 0xba, 0x2d, 0x75, 0x0f, 0xda, 0xb9, 0x72, 0x2d,
-	0x7e, 0x3c, 0x81, 0xb9, 0x4f, 0xbf, 0x81, 0x8e, 0x3a, 0x27, 0x77, 0x7b, 0x53, 0x25, 0x85, 0xd7,
+	0xf5, 0xae, 0x33, 0x3e, 0xda, 0x6d, 0x07, 0x40, 0xfd, 0xab, 0x89, 0x46, 0x69, 0x11, 0x56, 0x8c,
-	0x3f, 0x3e, 0xb1, 0xac, 0x70, 0x49, 0x73, 0xae, 0x7d, 0xde, 0x38, 0xfa, 0x00, 0x9f, 0xcc, 0x92,
+	0xc7, 0x62, 0x5a, 0x25, 0xa0, 0x83, 0x9f, 0xea, 0xc7, 0xe2, 0xdb, 0x3d, 0x98, 0xeb, 0xf4, 0x73,
-	0x88, 0xa6, 0x24, 0x14, 0xf9, 0x0d, 0x84, 0xbf, 0x4d, 0xe2, 0xec, 0xff, 0x05, 0x7d, 0x02, 0x1d,
+	0xe8, 0xe2, 0x38, 0x28, 0xfe, 0x4f, 0xa1, 0x85, 0x4e, 0x21, 0xfd, 0x07, 0x7b, 0x96, 0x11, 0x57,
-	0xa3, 0xbf, 0x0d, 0x68, 0x7e, 0xcb, 0x76, 0x89, 0xe0, 0xe8, 0x25, 0x98, 0x4b, 0x1a, 0x13, 0xae,
+	0x34, 0xe7, 0xbb, 0xcf, 0x13, 0xdd, 0x17, 0x70, 0x3c, 0x89, 0x67, 0x3c, 0x61, 0x61, 0x9a, 0xcf,
-	0xbe, 0x1d, 0x13, 0xeb, 0x44, 0xb2, 0xae, 0x87, 0xb3, 0x94, 0x12, 0xae, 0xcc, 0x61, 0xe2, 0x2a,
+	0xc0, 0xe4, 0xd3, 0x38, 0xca, 0xfe, 0x7f, 0xa1, 0x6f, 0x41, 0x87, 0xfb, 0x97, 0x01, 0xad, 0x2f,
-	0xa4, 0xbc, 0xa9, 0x9d, 0xc6, 0xd5, 0x27, 0x66, 0xe2, 0x32, 0x7f, 0xfc, 0x0c, 0x98, 0x27, 0xb6,
+	0xc5, 0x26, 0x4e, 0x25, 0x79, 0x08, 0xcd, 0x39, 0x8f, 0x98, 0xc4, 0x7f, 0xa7, 0x49, 0xb5, 0xa3,
-	0x5f, 0x82, 0x19, 0x64, 0x82, 0x14, 0xdf, 0x9e, 0x4e, 0x1e, 0xf8, 0xbc, 0xf9, 0xc8, 0xe7, 0x03,
+	0x58, 0xd7, 0xc5, 0x45, 0xc2, 0x99, 0x44, 0x71, 0x34, 0x69, 0x15, 0x42, 0x6d, 0x6a, 0xa5, 0x49,
-	0x68, 0xeb, 0x67, 0x67, 0xf6, 0x46, 0x39, 0xbc, 0x8b, 0xcb, 0x1c, 0x39, 0x50, 0xf1, 0x81, 0xa2,
+	0xfc, 0xc5, 0x9a, 0xb4, 0xf4, 0xab, 0x6c, 0x37, 0x30, 0x54, 0xb2, 0xfd, 0x10, 0x9a, 0x41, 0x96,
-	0xe2, 0x81, 0x33, 0x46, 0x6f, 0xa1, 0xa3, 0xaf, 0x9c, 0x13, 0x81, 0x3c, 0x68, 0x86, 0x2a, 0xc9,
+	0xb2, 0xe2, 0xdf, 0xd3, 0xce, 0x2d, 0x9d, 0xb7, 0x0e, 0x74, 0x7e, 0x0a, 0x1d, 0x7d, 0x68, 0x26,
-	0x45, 0x00, 0xf9, 0xe0, 0xe8, 0x72, 0xc1, 0xbd, 0xae, 0xcb, 0xf5, 0xc3, 0x94, 0xc8, 0x87, 0x45,
+	0xe7, 0xa8, 0xf0, 0x1e, 0x2d, 0x7d, 0x62, 0x43, 0x45, 0x07, 0x48, 0xc5, 0x2d, 0x65, 0xb8, 0x4f,
-	0x1d, 0x5e, 0xc7, 0x45, 0x3a, 0x75, 0xef, 0xfe, 0x72, 0x6a, 0x77, 0x07, 0xc7, 0xb8, 0x3f, 0x38,
+	0xa1, 0xab, 0xa7, 0xbc, 0x60, 0x29, 0xf1, 0xa0, 0x15, 0xa2, 0x93, 0x2f, 0x01, 0xd4, 0xfd, 0xd1,
-	0xc6, 0x9f, 0x07, 0xa7, 0xf6, 0xcb, 0xd1, 0xa9, 0xdd, 0x1e, 0x1d, 0xe3, 0xfe, 0xe8, 0xd4, 0x7e,
+	0xe1, 0x82, 0x7b, 0x1d, 0x57, 0xed, 0x87, 0x09, 0xf3, 0x8b, 0xbb, 0x58, 0xa7, 0x85, 0xeb, 0xfe,
-	0x3f, 0x3a, 0xb5, 0xa0, 0xa9, 0x94, 0x7d, 0xfd, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3a, 0x45,
+	0x64, 0xc0, 0x71, 0xe5, 0x64, 0x9d, 0xb3, 0x75, 0xc2, 0xf4, 0x05, 0x7a, 0xf7, 0xeb, 0x7b, 0x02,
-	0x1c, 0xc5, 0xce, 0x05, 0x00, 0x00,
+	0x2d, 0x3d, 0x08, 0x16, 0xe9, 0xd1, 0xdc, 0x53, 0xd5, 0x0b, 0x41, 0xd6, 0xb5, 0x54, 0x0b, 0x01,
+	0x1e, 0xd0, 0xba, 0x17, 0xb1, 0xfb, 0x3d, 0x1c, 0x57, 0x8e, 0x6d, 0xa5, 0xad, 0xc7, 0xff, 0x39,
+	0xbb, 0xef, 0x1d, 0x9c, 0xdd, 0x7d, 0x72, 0xde, 0xe0, 0xc1, 0xc5, 0x1d, 0x3b, 0xd7, 0x7f, 0xda,
+	0xb5, 0xeb, 0x9d, 0x6d, 0xbc, 0xd9, 0xd9, 0xc6, 0x1f, 0x3b, 0xbb, 0xf6, 0xe3, 0x8d, 0x5d, 0x7b,
+	0x7d, 0x63, 0x1b, 0x6f, 0x6e, 0xec, 0xda, 0x6f, 0x37, 0x76, 0x2d, 0x68, 0xe1, 0xa4, 0x8f, 0xfe,
+	0x0d, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xe1, 0xbd, 0x08, 0xe2, 0x06, 0x00, 0x00,
 }
 
 func (m *FileVersion) Marshal() (dAtA []byte, err error) {
@@ -403,6 +486,24 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if len(m.InvalidDevices) > 0 {
+		for iNdEx := len(m.InvalidDevices) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.InvalidDevices[iNdEx])
+			copy(dAtA[i:], m.InvalidDevices[iNdEx])
+			i = encodeVarintStructs(dAtA, i, uint64(len(m.InvalidDevices[iNdEx])))
+			i--
+			dAtA[i] = 0x22
+		}
+	}
+	if len(m.Devices) > 0 {
+		for iNdEx := len(m.Devices) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.Devices[iNdEx])
+			copy(dAtA[i:], m.Devices[iNdEx])
+			i = encodeVarintStructs(dAtA, i, uint64(len(m.Devices[iNdEx])))
+			i--
+			dAtA[i] = 0x1a
+		}
+	}
 	if m.Deleted {
 		i--
 		if m.Deleted {
@@ -411,24 +512,7 @@ func (m *FileVersion) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 			dAtA[i] = 0
 		}
 		i--
-		dAtA[i] = 0x20
+		dAtA[i] = 0x10
-	}
-	if m.Invalid {
-		i--
-		if m.Invalid {
-			dAtA[i] = 1
-		} else {
-			dAtA[i] = 0
-		}
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Device) > 0 {
-		i -= len(m.Device)
-		copy(dAtA[i:], m.Device)
-		i = encodeVarintStructs(dAtA, i, uint64(len(m.Device)))
-		i--
-		dAtA[i] = 0x12
 	}
 	{
 		size, err := m.Version.MarshalToSizedBuffer(dAtA[:i])
@@ -463,10 +547,10 @@ func (m *VersionList) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if len(m.Versions) > 0 {
+	if len(m.RawVersions) > 0 {
-		for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- {
+		for iNdEx := len(m.RawVersions) - 1; iNdEx >= 0; iNdEx-- {
 			{
-				size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				size, err := m.RawVersions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
 				if err != nil {
 					return 0, err
 				}
@@ -813,6 +897,103 @@ func (m *CountsSet) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *FileVersionDeprecated) Marshal() (dAtA []byte, err error) {
+	size := m.ProtoSize()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *FileVersionDeprecated) MarshalTo(dAtA []byte) (int, error) {
+	size := m.ProtoSize()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *FileVersionDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Deleted {
+		i--
+		if m.Deleted {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.Invalid {
+		i--
+		if m.Invalid {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Device) > 0 {
+		i -= len(m.Device)
+		copy(dAtA[i:], m.Device)
+		i = encodeVarintStructs(dAtA, i, uint64(len(m.Device)))
+		i--
+		dAtA[i] = 0x12
+	}
+	{
+		size, err := m.Version.MarshalToSizedBuffer(dAtA[:i])
+		if err != nil {
+			return 0, err
+		}
+		i -= size
+		i = encodeVarintStructs(dAtA, i, uint64(size))
+	}
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+func (m *VersionListDeprecated) Marshal() (dAtA []byte, err error) {
+	size := m.ProtoSize()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *VersionListDeprecated) MarshalTo(dAtA []byte) (int, error) {
+	size := m.ProtoSize()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *VersionListDeprecated) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for iNdEx := len(m.Versions) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Versions[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintStructs(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintStructs(dAtA []byte, offset int, v uint64) int {
 	offset -= sovStructs(v)
 	base := offset
@@ -832,16 +1013,21 @@ func (m *FileVersion) ProtoSize() (n int) {
 	_ = l
 	l = m.Version.ProtoSize()
 	n += 1 + l + sovStructs(uint64(l))
-	l = len(m.Device)
-	if l > 0 {
-		n += 1 + l + sovStructs(uint64(l))
-	}
-	if m.Invalid {
-		n += 2
-	}
 	if m.Deleted {
 		n += 2
 	}
+	if len(m.Devices) > 0 {
+		for _, b := range m.Devices {
+			l = len(b)
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	if len(m.InvalidDevices) > 0 {
+		for _, b := range m.InvalidDevices {
+			l = len(b)
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
 	return n
 }
 
@@ -851,8 +1037,8 @@ func (m *VersionList) ProtoSize() (n int) {
 	}
 	var l int
 	_ = l
-	if len(m.Versions) > 0 {
+	if len(m.RawVersions) > 0 {
-		for _, e := range m.Versions {
+		for _, e := range m.RawVersions {
 			l = e.ProtoSize()
 			n += 1 + l + sovStructs(uint64(l))
 		}
@@ -1007,6 +1193,42 @@ func (m *CountsSet) ProtoSize() (n int) {
 	return n
 }
 
+func (m *FileVersionDeprecated) ProtoSize() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	l = len(m.Device)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	if m.Invalid {
+		n += 2
+	}
+	if m.Deleted {
+		n += 2
+	}
+	return n
+}
+
+func (m *VersionListDeprecated) ProtoSize() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, e := range m.Versions {
+			l = e.ProtoSize()
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	return n
+}
+
 func sovStructs(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -1076,8 +1298,28 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
 			}
 			iNdEx = postIndex
 		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
+		case 3:
 			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType)
 			}
 			var byteLen int
 			for shift := uint(0); ; shift += 7 {
@@ -1104,36 +1346,14 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...)
+			m.Devices = append(m.Devices, make([]byte, postIndex-iNdEx))
-			if m.Device == nil {
+			copy(m.Devices[len(m.Devices)-1], dAtA[iNdEx:postIndex])
-				m.Device = []byte{}
-			}
 			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
-			}
-			var v int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowStructs
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				v |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			m.Invalid = bool(v != 0)
 		case 4:
-			if wireType != 0 {
+			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field InvalidDevices", wireType)
 			}
-			var v int
+			var byteLen int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
 					return ErrIntOverflowStructs
@@ -1143,12 +1363,24 @@ func (m *FileVersion) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				v |= int(b&0x7F) << shift
+				byteLen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
 			}
-			m.Deleted = bool(v != 0)
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.InvalidDevices = append(m.InvalidDevices, make([]byte, postIndex-iNdEx))
+			copy(m.InvalidDevices[len(m.InvalidDevices)-1], dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipStructs(dAtA[iNdEx:])
@@ -1204,7 +1436,7 @@ func (m *VersionList) Unmarshal(dAtA []byte) error {
 		switch fieldNum {
 		case 1:
 			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field RawVersions", wireType)
 			}
 			var msglen int
 			for shift := uint(0); ; shift += 7 {
@@ -1231,8 +1463,8 @@ func (m *VersionList) Unmarshal(dAtA []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Versions = append(m.Versions, FileVersion{})
+			m.RawVersions = append(m.RawVersions, FileVersion{})
-			if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+			if err := m.RawVersions[len(m.RawVersions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
 				return err
 			}
 			iNdEx = postIndex
@@ -2243,6 +2475,253 @@ func (m *CountsSet) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *FileVersionDeprecated) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileVersionDeprecated: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileVersionDeprecated: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Device = append(m.Device[:0], dAtA[iNdEx:postIndex]...)
+			if m.Device == nil {
+				m.Device = []byte{}
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Invalid = bool(v != 0)
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *VersionListDeprecated) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: VersionListDeprecated: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: VersionListDeprecated: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Versions = append(m.Versions, FileVersionDeprecated{})
+			if err := m.Versions[len(m.Versions)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipStructs(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

lib/db/structs.proto

@@ -13,15 +13,15 @@ option (gogoproto.goproto_unrecognized_all) = false;
 option (gogoproto.goproto_sizecache_all) = false;
 
 message FileVersion {
-    protocol.Vector version = 1 [(gogoproto.nullable) = false];
+    protocol.Vector version         = 1 [(gogoproto.nullable) = false];
-    bytes           device  = 2;
+    bool            deleted         = 2;
-    bool            invalid = 3;
+    repeated bytes  devices         = 3;
-    bool            deleted = 4;
+    repeated bytes  invalid_devices = 4;
 }
 
 message VersionList {
     option (gogoproto.goproto_stringer) = false;
-    repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
+    repeated FileVersion versions = 1 [(gogoproto.customname) = "RawVersions", (gogoproto.nullable) = false];
 }
 
 // Must be the same as FileInfo but without the blocks field
@@ -79,3 +79,15 @@ message CountsSet {
     repeated Counts counts  = 1  [(gogoproto.nullable) = false];
     int64           created = 2; // unix nanos
 }
+
+message FileVersionDeprecated {
+    protocol.Vector version = 1 [(gogoproto.nullable) = false];
+    bytes           device  = 2;
+    bool            invalid = 3;
+    bool            deleted = 4;
+}
+
+message VersionListDeprecated {
+    option (gogoproto.goproto_stringer) = false;
+    repeated FileVersionDeprecated versions = 1 [(gogoproto.nullable) = false];
+}

lib/db/transactions.go

@@ -15,7 +15,11 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-var errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry")
+var (
+	errEntryFromGlobalMissing = errors.New("device present in global list but missing as device/fileinfo entry")
+	errEmptyGlobal            = errors.New("no versions in global list")
+	errEmptyFileVersion       = errors.New("no devices in global file version")
+)
 
 // A readOnlyTransaction represents a database snapshot.
 type readOnlyTransaction struct {
@@ -54,7 +58,7 @@ func (t readOnlyTransaction) getFileByKey(key []byte) (protocol.FileInfo, bool,
 	return f.(protocol.FileInfo), true, nil
 }
 
-func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, bool, error) {
+func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (protocol.FileIntf, bool, error) {
 	bs, err := t.Get(key)
 	if backend.IsNotFound(err) {
 		return nil, false, nil
@@ -72,7 +76,7 @@ func (t readOnlyTransaction) getFileTrunc(key []byte, trunc bool) (FileIntf, boo
 	return f, true, nil
 }
 
-func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (FileIntf, error) {
+func (t readOnlyTransaction) unmarshalTrunc(bs []byte, trunc bool) (protocol.FileIntf, error) {
 	if trunc {
 		var tf FileInfoTruncated
 		err := tf.Unmarshal(bs)
@@ -175,26 +179,44 @@ func (t readOnlyTransaction) getGlobalVersionsByKey(key []byte) (VersionList, er
 	return vl, nil
 }
 
-func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, FileIntf, bool, error) {
+func (t readOnlyTransaction) getGlobal(keyBuf, folder, file []byte, truncate bool) ([]byte, protocol.FileIntf, bool, error) {
 	vl, err := t.getGlobalVersions(keyBuf, folder, file)
 	if backend.IsNotFound(err) {
 		return keyBuf, nil, false, nil
 	} else if err != nil {
 		return nil, nil, false, err
 	}
-	if len(vl.Versions) == 0 {
+	var fi protocol.FileIntf
-		return nil, nil, false, nil
+	keyBuf, fi, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, truncate, vl)
-	}
+	return keyBuf, fi, true, err
+}
 
-	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, vl.Versions[0].Device, file)
+func (t readOnlyTransaction) getGlobalFromVersionList(keyBuf, folder, file []byte, truncate bool, vl VersionList) ([]byte, protocol.FileIntf, FileVersion, error) {
+	fv, ok := vl.GetGlobal()
+	if !ok {
+		return keyBuf, nil, FileVersion{}, errEmptyGlobal
+	}
+	keyBuf, fi, err := t.getGlobalFromFileVersion(keyBuf, folder, file, truncate, fv)
+	return keyBuf, fi, fv, err
+}
+
+func (t readOnlyTransaction) getGlobalFromFileVersion(keyBuf, folder, file []byte, truncate bool, fv FileVersion) ([]byte, protocol.FileIntf, error) {
+	dev, ok := fv.FirstDevice()
+	if !ok {
+		return keyBuf, nil, errEmptyFileVersion
+	}
+	keyBuf, err := t.keyer.GenerateDeviceFileKey(keyBuf, folder, dev, file)
 	if err != nil {
-		return nil, nil, false, err
+		return keyBuf, nil, err
 	}
 	fi, ok, err := t.getFileTrunc(keyBuf, truncate)
-	if err != nil || !ok {
+	if err != nil {
-		return keyBuf, nil, false, err
+		return keyBuf, nil, err
 	}
-	return keyBuf, fi, true, nil
+	if !ok {
+		return keyBuf, nil, errEntryFromGlobalMissing
+	}
+	return keyBuf, fi, nil
 }
 
 func (t *readOnlyTransaction) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) error {
@@ -320,19 +342,12 @@ func (t *readOnlyTransaction) withGlobal(folder, prefix []byte, truncate bool, f
 			return err
 		}
 
-		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, vl.Versions[0].Device, name)
+		var f protocol.FileIntf
+		dk, f, _, err = t.getGlobalFromVersionList(dk, folder, name, truncate, vl)
 		if err != nil {
 			return err
 		}
 
-		f, ok, err := t.getFileTrunc(dk, truncate)
-		if err != nil {
-			return err
-		}
-		if !ok {
-			continue
-		}
-
 		if !fn(f) {
 			return nil
 		}
@@ -393,16 +408,13 @@ func (t *readOnlyTransaction) availability(folder, file []byte) ([]protocol.Devi
 		return nil, err
 	}
 
-	var devices []protocol.DeviceID
+	fv, ok := vl.GetGlobal()
-	for _, v := range vl.Versions {
+	if !ok {
-		if !v.Version.Equal(vl.Versions[0].Version) {
+		return nil, nil
-			break
+	}
-		}
+	devices := make([]protocol.DeviceID, len(fv.Devices))
-		if v.Invalid {
+	for i, dev := range fv.Devices {
-			continue
+		devices[i] = protocol.DeviceIDFromBytes(dev)
-		}
-		n := protocol.DeviceIDFromBytes(v.Device)
-		devices = append(devices, n)
 	}
 
 	return devices, nil
@@ -431,25 +443,28 @@ func (t *readOnlyTransaction) withNeed(folder, device []byte, truncate bool, fn
 			return err
 		}
 
-		globalFV := vl.Versions[0]
+		globalFV, ok := vl.GetGlobal()
+		if !ok {
+			return errEmptyGlobal
+		}
 		haveFV, have := vl.Get(device)
 
 		if !need(globalFV, have, haveFV.Version) {
 			continue
 		}
+
 		name := t.keyer.NameFromGlobalVersionKey(dbi.Key())
-		dk, err = t.keyer.GenerateDeviceFileKey(dk, folder, globalFV.Device, name)
+		var gf protocol.FileIntf
-		if err != nil {
+		dk, gf, err = t.getGlobalFromFileVersion(dk, folder, name, truncate, globalFV)
-			return err
-		}
-		gf, ok, err := t.getFileTrunc(dk, truncate)
 		if err != nil {
 			return err
 		}
+
+		globalDev, ok := globalFV.FirstDevice()
 		if !ok {
-			return errEntryFromGlobalMissing
+			return errEmptyFileVersion
 		}
-		l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.Invalid, haveFV.Version, globalFV.Version, globalFV.Device)
+		l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, devID, name, have, haveFV.IsInvalid(), haveFV.Version, gf.FileVersion(), globalDev)
 		if !fn(gf) {
 			return dbi.Error()
 		}
@@ -469,7 +484,7 @@ func (t *readOnlyTransaction) withNeedLocal(folder []byte, truncate bool, fn Ite
 	defer dbi.Release()
 
 	var keyBuf []byte
-	var f FileIntf
+	var f protocol.FileIntf
 	var ok bool
 	for dbi.Next() {
 		keyBuf, f, ok, err = t.getGlobal(keyBuf, folder, t.keyer.NameFromGlobalVersionKey(dbi.Key()), truncate)
@@ -586,7 +601,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 		return nil, false, err
 	}
 
-	fl, removedFV, removedAt, insertedAt, err := fl.update(folder, device, file, t.readOnlyTransaction)
+	globalFV, oldGlobalFV, removedFV, haveOldGlobal, haveRemoved, globalChanged, err := fl.update(folder, device, file, t.readOnlyTransaction)
 	if err != nil {
 		return nil, false, err
 	}
@@ -601,26 +616,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 	// Only load those from db if actually needed
 
 	var gotGlobal, gotOldGlobal bool
-	var global, oldGlobal FileIntf
+	var global, oldGlobal protocol.FileIntf
-
-	globalFV := fl.Versions[0]
-	var oldGlobalFV FileVersion
-	haveOldGlobal := false
-
-	globalUnaffected := removedAt != 0 && insertedAt != 0
-	if globalUnaffected {
-		oldGlobalFV = globalFV
-		haveOldGlobal = true
-	} else {
-		if removedAt == 0 {
-			oldGlobalFV = removedFV
-			haveOldGlobal = true
-		} else if len(fl.Versions) > 1 {
-			// The previous newest version is now at index 1
-			oldGlobalFV = fl.Versions[1]
-			haveOldGlobal = true
-		}
-	}
 
 	// Check the need of the device that was updated
 	// Must happen before updating global meta: If this is the first
@@ -628,11 +624,11 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 
 	needBefore := false
 	if haveOldGlobal {
-		needBefore = need(oldGlobalFV, removedAt >= 0, removedFV.Version)
+		needBefore = need(oldGlobalFV, haveRemoved, removedFV.Version)
 	}
-	needNow := need(globalFV, true, fl.Versions[insertedAt].Version)
+	needNow := need(globalFV, true, file.Version)
 	if needBefore {
-		if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
+		if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil {
 			return nil, false, err
 		}
 		gotOldGlobal = true
@@ -644,7 +640,7 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 		}
 	}
 	if needNow {
-		if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
+		if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil {
 			return nil, false, err
 		}
 		gotGlobal = true
@@ -657,42 +653,34 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 	}
 
 	// Update global size counter if necessary
-	// Necessary here means the first item in the global list was changed,
-	// even if both new and old are invalid, due to potential change in
-	// LocalFlags.
 
-	// Neither the global state nor the needs of any devices, except the one
+	if !globalChanged {
-	// updated, changed.
+		// Neither the global state nor the needs of any devices, except
-	if globalUnaffected {
+		// the one updated, changed.
 		return keyBuf, true, nil
 	}
 
 	// Remove the old global from the global size counter
 	if haveOldGlobal {
 		if !gotOldGlobal {
-			if oldGlobal, err = t.updateGlobalGetOldGlobal(keyBuf, folder, name, oldGlobalFV); err != nil {
+			if keyBuf, oldGlobal, err = t.getGlobalFromFileVersion(keyBuf, folder, name, true, oldGlobalFV); err != nil {
 				return nil, false, err
 			}
 			gotOldGlobal = true
 		}
+		// Remove the old global from the global size counter
 		meta.removeFile(protocol.GlobalDeviceID, oldGlobal)
 	}
 
 	// Add the new global to the global size counter
 	if !gotGlobal {
-		if global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, insertedAt, fl); err != nil {
+		if keyBuf, global, err = t.updateGlobalGetGlobal(keyBuf, folder, name, file, globalFV); err != nil {
 			return nil, false, err
 		}
 		gotGlobal = true
 	}
 	meta.addFile(protocol.GlobalDeviceID, global)
 
-	// If global changed, but both the new and old are invalid, noone needed
-	// the file before and now -> nothing to do.
-	if global.IsInvalid() && (!haveOldGlobal || oldGlobal.IsInvalid()) {
-		return keyBuf, true, nil
-	}
-
 	// check for local (if not already done before)
 	if !bytes.Equal(device, protocol.LocalDeviceID[:]) {
 		localFV, haveLocal := fl.Get(protocol.LocalDeviceID[:])
@@ -736,40 +724,12 @@ func (t readWriteTransaction) updateGlobal(gk, keyBuf, folder, device []byte, fi
 	return keyBuf, true, nil
 }
 
-func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, insertedAt int, fl VersionList) (FileIntf, error) {
+func (t readWriteTransaction) updateGlobalGetGlobal(keyBuf, folder, name []byte, file protocol.FileInfo, fv FileVersion) ([]byte, protocol.FileIntf, error) {
-	if insertedAt == 0 {
+	if fv.Version.Equal(file.Version) {
 		// Inserted a new newest version
-		return file, nil
+		return keyBuf, file, nil
 	}
-	var err error
+	return t.getGlobalFromFileVersion(keyBuf, folder, name, true, fv)
-	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, name)
-	if err != nil {
-		return nil, err
-	}
-	global, ok, err := t.getFileTrunc(keyBuf, true)
-	if err != nil {
-		return nil, err
-	}
-	if !ok {
-		return nil, errEntryFromGlobalMissing
-	}
-	return global, nil
-}
-
-func (t readWriteTransaction) updateGlobalGetOldGlobal(keyBuf, folder, name []byte, oldGlobalFV FileVersion) (FileIntf, error) {
-	var err error
-	keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, oldGlobalFV.Device, name)
-	if err != nil {
-		return nil, err
-	}
-	oldGlobal, ok, err := t.getFileTrunc(keyBuf, true)
-	if err != nil {
-		return nil, err
-	}
-	if !ok {
-		return nil, errEntryFromGlobalMissing
-	}
-	return oldGlobal, nil
 }
 
 func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add bool) ([]byte, error) {
@@ -790,7 +750,7 @@ func (t readWriteTransaction) updateLocalNeed(keyBuf, folder, name []byte, add b
 
 func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool {
 	// We never need an invalid file.
-	if global.Invalid {
+	if global.IsInvalid() {
 		return false
 	}
 	// We don't need a deleted file if we don't have it.
@@ -807,7 +767,7 @@ func need(global FileVersion, haveLocal bool, localVersion protocol.Vector) bool
 // removeFromGlobal removes the device from the global version list for the
 // given file. If the version list is empty after this, the file entry is
 // removed entirely.
-func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte, file []byte, meta *metadataTracker) ([]byte, error) {
+func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device, file []byte, meta *metadataTracker) ([]byte, error) {
 	deviceID := protocol.DeviceIDFromBytes(device)
 
 	l.Debugf("remove from global; folder=%q device=%v file=%q", folder, deviceID, file)
@@ -821,34 +781,32 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
 		return nil, err
 	}
 
-	if len(fl.Versions) == 0 {
+	oldGlobalFV, haveOldGlobal := fl.GetGlobal()
+
+	if !haveOldGlobal {
 		// Shouldn't ever happen, but doesn't hurt to handle.
 		return keyBuf, t.Delete(gk)
 	}
 
-	oldGlobalFV := fl.Versions[0]
+	removedFV, haveRemoved, globalChanged, err := fl.pop(folder, device, file, t.readOnlyTransaction)
-
+	if err != nil {
-	fl, removedFV, removedAt := fl.pop(device)
+		return nil, err
-	if removedAt == -1 {
+	}
+	if !haveRemoved {
 		// There is no version for the given device
 		return keyBuf, nil
 	}
 
-	var global FileIntf
+	var global protocol.FileIntf
 	var gotGlobal, ok bool
 
+	globalFV, ok := fl.GetGlobal()
 	// Add potential needs of the removed device
-	if len(fl.Versions) != 0 && !fl.Versions[0].Invalid && need(fl.Versions[0], false, protocol.Vector{}) && !need(oldGlobalFV, removedAt != -1, removedFV.Version) {
+	if ok && !globalFV.IsInvalid() && need(globalFV, false, protocol.Vector{}) && !need(oldGlobalFV, haveRemoved, removedFV.Version) {
-		keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
+		keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl)
 		if err != nil {
 			return nil, err
 		}
-		global, ok, err = t.getFileTrunc(keyBuf, true)
-		if err != nil {
-			return nil, err
-		} else if !ok {
-			return nil, errEntryFromGlobalMissing
-		}
 		gotGlobal = true
 		meta.addNeeded(deviceID, global)
 		if bytes.Equal(protocol.LocalDeviceID[:], device) {
@@ -859,7 +817,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
 	}
 
 	// Global hasn't changed, abort early
-	if removedAt != 0 {
+	if !globalChanged {
 		l.Debugf("new global after remove: %v", fl)
 		if err := t.Put(gk, mustMarshal(&fl)); err != nil {
 			return nil, err
@@ -896,7 +854,7 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
 	}
 
 	// Nothing left, i.e. nothing to add to the global counter below.
-	if len(fl.Versions) == 0 {
+	if fl.Empty() {
 		if err := t.Delete(gk); err != nil {
 			return nil, err
 		}
@@ -905,21 +863,14 @@ func (t readWriteTransaction) removeFromGlobal(gk, keyBuf, folder, device []byte
 
 	// Add to global
 	if !gotGlobal {
-		keyBuf, err = t.keyer.GenerateDeviceFileKey(keyBuf, folder, fl.Versions[0].Device, file)
+		keyBuf, global, _, err = t.getGlobalFromVersionList(keyBuf, folder, file, true, fl)
 		if err != nil {
 			return nil, err
 		}
-		global, ok, err = t.getFileTrunc(keyBuf, true)
-		if err != nil {
-			return nil, err
-		}
-		if !ok {
-			return nil, errEntryFromGlobalMissing
-		}
 	}
 	meta.addFile(protocol.GlobalDeviceID, global)
 
-	l.Debugf("new global after remove: %v", fl)
+	l.Debugf(`new global for "%s" after remove: %v`, file, fl)
 	if err := t.Put(gk, mustMarshal(&fl)); err != nil {
 		return nil, err
 	}

lib/model/folder.go

@@ -303,7 +303,7 @@ func (f *folder) pull() (success bool) {
 	// If there is nothing to do, don't even enter sync-waiting state.
 	abort := true
 	snap := f.fset.Snapshot()
-	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
 		abort = false
 		return false
 	})
@@ -499,7 +499,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 	for _, sub := range subDirs {
 		var iterError error
 
-		snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
+		snap.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi protocol.FileIntf) bool {
 			select {
 			case <-f.ctx.Done():
 				return false
@@ -634,7 +634,7 @@ func (f *folder) findRename(snap *db.Snapshot, mtimefs fs.Filesystem, file proto
 	found := false
 	nf := protocol.FileInfo{}
 
-	snap.WithBlocksHash(file.BlocksHash, func(ifi db.FileIntf) bool {
+	snap.WithBlocksHash(file.BlocksHash, func(ifi protocol.FileIntf) bool {
 		fi := ifi.(protocol.FileInfo)
 
 		select {

lib/model/folder_recvonly.go

@@ -87,7 +87,7 @@ func (f *receiveOnlyFolder) revert() {
 	batchSizeBytes := 0
 	snap := f.fset.Snapshot()
 	defer snap.Release()
-	snap.WithHave(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
 		fi := intf.(protocol.FileInfo)
 		if !fi.IsReceiveOnlyChanged() {
 			// We're only interested in files that have changed locally in

lib/model/folder_sendonly.go

@@ -52,7 +52,7 @@ func (f *sendOnlyFolder) pull() bool {
 
 	snap := f.fset.Snapshot()
 	defer snap.Release()
-	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
 			f.updateLocalsFromPulling(batch)
 			batch = batch[:0]
@@ -110,7 +110,7 @@ func (f *sendOnlyFolder) override() {
 	batchSizeBytes := 0
 	snap := f.fset.Snapshot()
 	defer snap.Release()
-	snap.WithNeed(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
+	snap.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
 		if len(batch) == maxBatchSizeFiles || batchSizeBytes > maxBatchSizeBytes {
 			f.updateLocalsFromScanning(batch)

lib/model/folder_sendrecv.go

@@ -305,7 +305,7 @@ func (f *sendReceiveFolder) processNeeded(snap *db.Snapshot, dbUpdateChan chan<-
 	// Regular files to pull goes into the file queue, everything else
 	// (directories, symlinks and deletes) goes into the "process directly"
 	// pile.
-	snap.WithNeed(protocol.LocalDeviceID, func(intf db.FileIntf) bool {
+	snap.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
 		select {
 		case <-f.ctx.Done():
 			return false

lib/model/model.go

@@ -851,7 +851,7 @@ func (m *model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 	}
 
 	rest = make([]db.FileInfoTruncated, 0, perpage)
-	snap.WithNeedTruncated(protocol.LocalDeviceID, func(f db.FileIntf) bool {
+	snap.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
 		if cfg.IgnoreDelete && f.IsDeleted() {
 			return true
 		}
@@ -1936,7 +1936,7 @@ func (s *indexSender) sendIndexTo(ctx context.Context) error {
 	snap := s.fset.Snapshot()
 	defer snap.Release()
 	previousWasDelete := false
-	snap.WithHaveSequence(s.prevSequence+1, func(fi db.FileIntf) bool {
+	snap.WithHaveSequence(s.prevSequence+1, func(fi protocol.FileIntf) bool {
 		// This is to make sure that renames (which is an add followed by a delete) land in the same batch.
 		// Even if the batch is full, we allow a last delete to slip in, we do this by making sure that
 		// the batch ends with a non-delete, or that the last item in the batch is already a delete
@@ -2248,7 +2248,7 @@ func (m *model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 
 	snap := files.Snapshot()
 	defer snap.Release()
-	snap.WithPrefixedGlobalTruncated(prefix, func(fi db.FileIntf) bool {
+	snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileIntf) bool {
 		f := fi.(db.FileInfoTruncated)
 
 		// Don't include the prefix itself.

lib/model/model_test.go

@@ -2426,7 +2426,7 @@ func TestIssue3496(t *testing.T) {
 	m.fmut.RUnlock()
 	var localFiles []protocol.FileInfo
 	snap := fs.Snapshot()
-	snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
 		localFiles = append(localFiles, i.(protocol.FileInfo))
 		return true
 	})
@@ -3556,7 +3556,7 @@ func TestRenameSequenceOrder(t *testing.T) {
 
 	count := 0
 	snap := dbSnapshot(t, m, "default")
-	snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
 		count++
 		return true
 	})
@@ -3588,7 +3588,7 @@ func TestRenameSequenceOrder(t *testing.T) {
 	var firstExpectedSequence int64
 	var secondExpectedSequence int64
 	failed := false
-	snap.WithHaveSequence(0, func(i db.FileIntf) bool {
+	snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
 		t.Log(i)
 		if i.FileName() == "17" {
 			firstExpectedSequence = i.SequenceNo() + 1
@@ -3621,7 +3621,7 @@ func TestRenameSameFile(t *testing.T) {
 
 	count := 0
 	snap := dbSnapshot(t, m, "default")
-	snap.WithHave(protocol.LocalDeviceID, func(i db.FileIntf) bool {
+	snap.WithHave(protocol.LocalDeviceID, func(i protocol.FileIntf) bool {
 		count++
 		return true
 	})
@@ -3644,7 +3644,7 @@ func TestRenameSameFile(t *testing.T) {
 
 	prevSeq := int64(0)
 	seen := false
-	snap.WithHaveSequence(0, func(i db.FileIntf) bool {
+	snap.WithHaveSequence(0, func(i protocol.FileIntf) bool {
 		if i.SequenceNo() <= prevSeq {
 			t.Fatalf("non-increasing sequences: %d <= %d", i.SequenceNo(), prevSeq)
 		}
@@ -3683,7 +3683,7 @@ func TestRenameEmptyFile(t *testing.T) {
 	}
 
 	count := 0
-	snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
+	snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
 		count++
 		return true
 	})
@@ -3693,7 +3693,7 @@ func TestRenameEmptyFile(t *testing.T) {
 	}
 
 	count = 0
-	snap.WithBlocksHash(file.BlocksHash, func(_ db.FileIntf) bool {
+	snap.WithBlocksHash(file.BlocksHash, func(_ protocol.FileIntf) bool {
 		count++
 		return true
 	})
@@ -3712,7 +3712,7 @@ func TestRenameEmptyFile(t *testing.T) {
 	defer snap.Release()
 
 	count = 0
-	snap.WithBlocksHash(empty.BlocksHash, func(_ db.FileIntf) bool {
+	snap.WithBlocksHash(empty.BlocksHash, func(_ protocol.FileIntf) bool {
 		count++
 		return true
 	})
@@ -3722,7 +3722,7 @@ func TestRenameEmptyFile(t *testing.T) {
 	}
 
 	count = 0
-	snap.WithBlocksHash(file.BlocksHash, func(i db.FileIntf) bool {
+	snap.WithBlocksHash(file.BlocksHash, func(i protocol.FileIntf) bool {
 		count++
 		if i.FileName() != "new-file" {
 			t.Fatalf("unexpected file name %s, expected new-file", i.FileName())
@@ -3757,7 +3757,7 @@ func TestBlockListMap(t *testing.T) {
 	}
 	var paths []string
 
-	snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
+	snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
 		paths = append(paths, fi.FileName())
 		return true
 	})
@@ -3790,7 +3790,7 @@ func TestBlockListMap(t *testing.T) {
 	defer snap.Release()
 
 	paths = paths[:0]
-	snap.WithBlocksHash(fi.BlocksHash, func(fi db.FileIntf) bool {
+	snap.WithBlocksHash(fi.BlocksHash, func(fi protocol.FileIntf) bool {
 		paths = append(paths, fi.FileName())
 		return true
 	})

lib/protocol/bep_extensions.go

@@ -23,6 +23,31 @@ const (
 	Version13HelloMagic    uint32 = 0x9F79BC40 // old
 )
 
+// FileIntf is the set of methods implemented by both FileInfo and
+// db.FileInfoTruncated.
+type FileIntf interface {
+	FileSize() int64
+	FileName() string
+	FileLocalFlags() uint32
+	IsDeleted() bool
+	IsInvalid() bool
+	IsIgnored() bool
+	IsUnsupported() bool
+	MustRescan() bool
+	IsReceiveOnlyChanged() bool
+	IsDirectory() bool
+	IsSymlink() bool
+	ShouldConflict() bool
+	HasPermissionBits() bool
+	SequenceNo() int64
+	BlockSize() int
+	FileVersion() Vector
+	FileType() FileInfoType
+	FilePermissions() uint32
+	FileModifiedBy() ShortID
+	ModTime() time.Time
+}
+
 func (m Hello) Magic() uint32 {
 	return HelloMessageMagic
 }
@@ -139,7 +164,7 @@ func (f FileInfo) FileModifiedBy() ShortID {
 
 // WinsConflict returns true if "f" is the one to choose when it is in
 // conflict with "other".
-func (f FileInfo) WinsConflict(other FileInfo) bool {
+func WinsConflict(f, other FileIntf) bool {
 	// If only one of the files is invalid, that one loses.
 	if f.IsInvalid() != other.IsInvalid() {
 		return !f.IsInvalid()
@@ -164,7 +189,7 @@ func (f FileInfo) WinsConflict(other FileInfo) bool {
 
 	// The modification times were equal. Use the device ID in the version
 	// vector as tie breaker.
-	return f.Version.Compare(other.Version) == ConcurrentGreater
+	return f.FileVersion().Compare(other.FileVersion()) == ConcurrentGreater
 }
 
 func (f FileInfo) IsEmpty() bool {

lib/protocol/conflict_test.go

@@ -14,10 +14,10 @@ func TestWinsConflict(t *testing.T) {
 	}
 
 	for _, tc := range testcases {
-		if !tc[0].WinsConflict(tc[1]) {
+		if !WinsConflict(tc[0], tc[1]) {
 			t.Errorf("%v should win over %v", tc[0], tc[1])
 		}
-		if tc[1].WinsConflict(tc[0]) {
+		if WinsConflict(tc[1], tc[0]) {
 			t.Errorf("%v should not win over %v", tc[1], tc[0])
 		}
 	}