Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/db: Add index to track locally needed files (#4958) 

To optimize WithNeed, which is called for the local device whenever an index
update is received. No tracking for remote devices to conserve db space, as
WithNeed is only queried for completion.
This commit is contained in:
Simon Frei 2018-06-02 15:08:32 +02:00 committed by Jakob Borg
parent d3a02a1663
commit 5baa432906
10 changed files with 440 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
lib/db/leveldb.go
View File

@ -13,7 +13,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
const dbVersion = 2
const dbVersion = 3
const (
KeyTypeDevice = iota
@ -28,13 +28,14 @@ const (
KeyTypeFolderMeta
KeyTypeMiscData
KeyTypeSequence
KeyTypeNeed
)
func (l VersionList) String() string {
func (vl VersionList) String() string {
var b bytes.Buffer
var id protocol.DeviceID
b.WriteString("{")
for i, v := range l.Versions {
for i, v := range vl.Versions {
if i > 0 {
b.WriteString(", ")
}
@ -45,18 +46,90 @@ func (l VersionList) String() string {
return b.String()
}
// update brings the VersionList up to date with file. It returns the updated
// VersionList, a potentially removed old FileVersion and its index, as well as
// the index where the new FileVersion was inserted.
func (vl VersionList) update(folder, device []byte, file protocol.FileInfo, db *Instance) (_ VersionList, removedFV FileVersion, removedAt int, insertedAt int) {
removedAt, insertedAt = -1, -1
for i, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
removedAt = i
removedFV = v
vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
break
}
}
nv := FileVersion{
Device: device,
Version: file.Version,
Invalid: file.Invalid,
}
for i, v := range vl.Versions {
switch v.Version.Compare(file.Version) {
case protocol.Equal:
if nv.Invalid {
continue
}
fallthrough
case protocol.Lesser:
// The version at this point in the list is equal to or lesser
// ("older") than us. We insert ourselves in front of it.
vl = vl.insertAt(i, nv)
return vl, removedFV, removedAt, i
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version at this point is in conflict with us. We must pull
// the actual file metadata to determine who wins. If we win, we
// insert ourselves in front of the loser here. (The "Lesser" and
// "Greater" in the condition above is just based on the device
// IDs in the version vector, which is not the only thing we use
// to determine the winner.)
//
// A surprise missing file entry here is counted as a win for us.
if of, ok := db.getFile(db.deviceKey(folder, v.Device, []byte(file.Name))); !ok || file.WinsConflict(of) {
vl = vl.insertAt(i, nv)
return vl, removedFV, removedAt, i
}
}
}
// We didn't find a position for an insert above, so append to the end.
vl.Versions = append(vl.Versions, nv)
return vl, removedFV, removedAt, len(vl.Versions) - 1
}
func (vl VersionList) insertAt(i int, v FileVersion) VersionList {
vl.Versions = append(vl.Versions, FileVersion{})
copy(vl.Versions[i+1:], vl.Versions[i:])
vl.Versions[i] = v
return vl
}
func (vl VersionList) Get(device []byte) (FileVersion, bool) {
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
return v, true
}
}
return FileVersion{}, false
}
type fileList []protocol.FileInfo
func (l fileList) Len() int {
return len(l)
func (fl fileList) Len() int {
return len(fl)
}
func (l fileList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
func (fl fileList) Swap(a, b int) {
fl[a], fl[b] = fl[b], fl[a]
}
func (l fileList) Less(a, b int) bool {
return l[a].Name < l[b].Name
func (fl fileList) Less(a, b int) bool {
return fl[a].Name < fl[b].Name
}
// Flush batches to disk when they contain this many records.

160
lib/db/leveldb_dbinstance.go
View File

@ -103,6 +103,9 @@ func (db *Instance) UpdateSchema() {
if prevVersion <= 1 {
db.updateSchema1to2()
}
if prevVersion <= 2 {
db.updateSchema2to3()
}
l.Infof("Finished updating database schema version from %v to %v", prevVersion, dbVersion)
miscDB.PutInt64("dbVersion", dbVersion)
@ -310,26 +313,32 @@ func (db *Instance) getFileTrunc(key []byte, trunc bool) (FileIntf, bool) {
}
func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, bool) {
k := db.globalKey(folder, file)
t := db.newReadOnlyTransaction()
defer t.close()
bs, err := t.Get(k, nil)
_, _, f, ok := db.getGlobalInto(t, nil, nil, folder, file, truncate)
return f, ok
}
func (db *Instance) getGlobalInto(t readOnlyTransaction, gk, dk, folder, file []byte, truncate bool) ([]byte, []byte, FileIntf, bool) {
gk = db.globalKeyInto(gk, folder, file)
bs, err := t.Get(gk, nil)
if err != nil {
return nil, false
return gk, dk, nil, false
}
vl, ok := unmarshalVersionList(bs)
if !ok {
return nil, false
return gk, dk, nil, false
}
if fi, ok := db.getFileTrunc(db.deviceKey(folder, vl.Versions[0].Device, file), truncate); ok {
return fi, true
dk = db.deviceKeyInto(dk, folder, vl.Versions[0].Device, file)
if fi, ok := db.getFileTrunc(dk, truncate); ok {
return gk, dk, fi, true
}
return nil, false
return gk, dk, nil, false
}
func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator) {
@ -409,6 +418,11 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
}
func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator) {
if bytes.Equal(device, protocol.LocalDeviceID[:]) {
db.withNeedLocal(folder, truncate, fn)
return
}
t := db.newReadOnlyTransaction()
defer t.close()
@ -422,22 +436,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
continue
}
have := false // If we have the file, any version
need := false // If we have a lower version of the file
var haveFileVersion FileVersion
for _, v := range vl.Versions {
if bytes.Equal(v.Device, device) {
have = true
haveFileVersion = v
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
need = !v.Version.GreaterEqual(vl.Versions[0].Version)
break
}
}
if have && !need {
haveFV, have := vl.Get(device)
// XXX: This marks Concurrent (i.e. conflicting) changes as
// needs. Maybe we should do that, but it needs special
// handling in the puller.
if have && haveFV.Version.GreaterEqual(vl.Versions[0].Version) {
continue
}
@ -474,7 +477,7 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
break
}
l.Debugf("need folder=%q device=%v name=%q need=%v have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveFileVersion.Invalid, haveFileVersion.Version, needVersion, needDevice)
l.Debugf("need folder=%q device=%v name=%q have=%v invalid=%v haveV=%v globalV=%v globalDev=%v", folder, protocol.DeviceIDFromBytes(device), name, have, haveFV.Invalid, haveFV.Version, needVersion, needDevice)
if !fn(gf) {
return
@ -486,6 +489,28 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
}
}
func (db *Instance) withNeedLocal(folder []byte, truncate bool, fn Iterator) {
t := db.newReadOnlyTransaction()
defer t.close()
dbi := t.NewIterator(util.BytesPrefix(db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen]), nil)
defer dbi.Release()
var dk []byte
var gk []byte
var f FileIntf
var ok bool
for dbi.Next() {
gk, dk, f, ok = db.getGlobalInto(t, gk, dk, folder, db.globalKeyName(dbi.Key()), truncate)
if !ok {
continue
}
if !fn(f) {
return
}
}
}
func (db *Instance) ListFolders() []string {
t := db.newReadOnlyTransaction()
defer t.close()
@ -511,36 +536,21 @@ func (db *Instance) ListFolders() []string {
}
func (db *Instance) dropFolder(folder []byte) {
t := db.newReadOnlyTransaction()
t := db.newReadWriteTransaction()
defer t.close()
// Remove all items related to the given folder from the device->file bucket
dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil)
for dbi.Next() {
itemFolder := db.deviceKeyFolder(dbi.Key())
if bytes.Equal(folder, itemFolder) {
db.Delete(dbi.Key(), nil)
}
for _, key := range [][]byte{
// Remove all items related to the given folder from the device->file bucket
db.deviceKey(folder, nil, nil)[:keyPrefixLen+keyFolderLen],
// Remove all sequences related to the folder
db.sequenceKey([]byte(folder), 0)[:keyPrefixLen+keyFolderLen],
// Remove all items related to the given folder from the global bucket
db.globalKey(folder, nil)[:keyPrefixLen+keyFolderLen],
// Remove all needs related to the folder
db.needKey(folder, nil)[:keyPrefixLen+keyFolderLen],
} {
t.deleteKeyPrefix(key)
}
dbi.Release()
// Remove all sequences related to the folder
sequenceKey := db.sequenceKey([]byte(folder), 0)
dbi = t.NewIterator(util.BytesPrefix(sequenceKey[:keyPrefixLen+keyFolderLen]), nil)
for dbi.Next() {
db.Delete(dbi.Key(), nil)
}
dbi.Release()
// Remove all items related to the given folder from the global bucket
dbi = t.NewIterator(util.BytesPrefix([]byte{KeyTypeGlobal}), nil)
for dbi.Next() {
itemFolder, ok := db.globalKeyFolder(dbi.Key())
if ok && bytes.Equal(folder, itemFolder) {
db.Delete(dbi.Key(), nil)
}
}
dbi.Release()
}
func (db *Instance) dropDeviceFolder(device, folder []byte, meta *metadataTracker) {
@ -680,13 +690,14 @@ func (db *Instance) updateSchema0to1() {
l.Infof("Updated symlink type for %d index entries and added %d invalid files to global list", symlinkConv, ignAdded)
}
// updateSchema1to2 introduces a sequenceKey->deviceKey bucket for local items
// to allow iteration in sequence order (simplifies sending indexes).
func (db *Instance) updateSchema1to2() {
t := db.newReadWriteTransaction()
defer t.close()
var sk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withHave(folder, protocol.LocalDeviceID[:], nil, true, func(f FileIntf) bool {
@ -699,6 +710,34 @@ func (db *Instance) updateSchema1to2() {
}
}
// updateSchema2to3 introduces a needKey->nil bucket for locally needed files.
func (db *Instance) updateSchema2to3() {
t := db.newReadWriteTransaction()
defer t.close()
var nk []byte
var dk []byte
for _, folderStr := range db.ListFolders() {
folder := []byte(folderStr)
db.withGlobal(folder, nil, true, func(f FileIntf) bool {
name := []byte(f.FileName())
dk = db.deviceKeyInto(dk, folder, protocol.LocalDeviceID[:], name)
var v protocol.Vector
haveFile, ok := db.getFileTrunc(dk, true)
if ok {
v = haveFile.FileVersion()
}
if !need(f, ok, v) {
return true
}
nk = t.db.needKeyInto(nk, folder, []byte(f.FileName()))
t.Put(nk, nil)
t.checkFlush()
return true
})
}
}
// deviceKey returns a byte slice encoding the following information:
// keyTypeDevice (1 byte)
// folder (4 bytes)
@ -755,7 +794,7 @@ func (db *Instance) globalKeyInto(gk, folder, file []byte) []byte {
gk[0] = KeyTypeGlobal
binary.BigEndian.PutUint32(gk[keyPrefixLen:], db.folderIdx.ID(folder))
copy(gk[keyPrefixLen+keyFolderLen:], file)
return gk[:reqLen]
return gk
}
// globalKeyName returns the filename from the key
@ -768,6 +807,17 @@ func (db *Instance) globalKeyFolder(key []byte) ([]byte, bool) {
return db.folderIdx.Val(binary.BigEndian.Uint32(key[keyPrefixLen:]))
}
// needKey is a globalKey with a different prefix
func (db *Instance) needKey(folder, file []byte) []byte {
return db.needKeyInto(nil, folder, file)
}
func (db *Instance) needKeyInto(k, folder, file []byte) []byte {
k = db.globalKeyInto(k, folder, file)
k[0] = KeyTypeNeed
return k
}
// sequenceKey returns a byte slice encoding the following information:
// KeyTypeSequence (1 byte)
// folder (4 bytes)
@ -782,7 +832,7 @@ func (db *Instance) sequenceKeyInto(k []byte, folder []byte, seq int64) []byte {
k[0] = KeyTypeSequence
binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder))
binary.BigEndian.PutUint64(k[keyPrefixLen+keyFolderLen:], uint64(seq))
return k[:reqLen]
return k
}
// sequenceKeySequence returns the sequence number from the key

96
lib/db/leveldb_test.go
View File

@ -224,3 +224,99 @@ func TestInvalidFiles(t *testing.T) {
t.Error("quux should not be invalid")
}
}
const myID = 1
var (
remoteDevice0, remoteDevice1 protocol.DeviceID
update0to3Folder = "UpdateSchema0to3"
invalid = "invalid"
slashPrefixed = "/notgood"
haveUpdate0to3 map[protocol.DeviceID]fileList
)
func init() {
remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
haveUpdate0to3 = map[protocol.DeviceID]fileList{
protocol.LocalDeviceID: {
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: slashPrefixed, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
},
remoteDevice0: {
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
},
remoteDevice1: {
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
protocol.FileInfo{Name: invalid, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
},
}
}
func TestUpdate0to3(t *testing.T) {
ldb, err := openJSONS("testdata/v0.14.45-update0to3.db.jsons")
if err != nil {
t.Fatal(err)
}
db := newDBInstance(ldb, "<memory>")
folder := []byte(update0to3Folder)
db.updateSchema0to1()
if _, ok := db.getFile(db.deviceKey(folder, protocol.LocalDeviceID[:], []byte(slashPrefixed))); ok {
t.Error("File prefixed by '/' was not removed during transition to schema 1")
}
if _, err := db.Get(db.globalKey(folder, []byte(invalid)), nil); err != nil {
t.Error("Invalid file wasn't added to global list")
}
db.updateSchema1to2()
found := false
db.withHaveSequence(folder, 0, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
l.Infoln(f)
if found {
t.Error("Unexpected additional file via sequence", f.FileName())
return true
}
if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalent(e, true, true) {
found = true
} else {
t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
}
return true
})
if !found {
t.Error("Local file wasn't added to sequence bucket", err)
}
db.updateSchema2to3()
need := map[string]protocol.FileInfo{
haveUpdate0to3[remoteDevice0][0].Name: haveUpdate0to3[remoteDevice0][0],
haveUpdate0to3[remoteDevice1][0].Name: haveUpdate0to3[remoteDevice1][0],
haveUpdate0to3[remoteDevice0][2].Name: haveUpdate0to3[remoteDevice0][2],
}
db.withNeed(folder, protocol.LocalDeviceID[:], false, func(fi FileIntf) bool {
e, ok := need[fi.FileName()]
if !ok {
t.Error("Got unexpected needed file:", fi.FileName())
}
f := fi.(protocol.FileInfo)
delete(need, f.Name)
if !f.IsEquivalent(e, true, true) {
t.Errorf("Wrong needed file, got %v, expected %v", f, e)
}
return true
})
for n := range need {
t.Errorf(`Missing needed file "%v"`, n)
}
}

181
lib/db/leveldb_transactions.go
View File

@ -12,6 +12,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
// A readOnlyTransaction represents a database snapshot.
@ -85,115 +86,87 @@ func (t readWriteTransaction) insertFile(fk, folder, device []byte, file protoco
// If the file does not have an entry in the global list, it is created.
func (t readWriteTransaction) updateGlobal(gk, folder, device []byte, file protocol.FileInfo, meta *metadataTracker) bool {
l.Debugf("update global; folder=%q device=%v file=%q version=%v invalid=%v", folder, protocol.DeviceIDFromBytes(device), file.Name, file.Version, file.Invalid)
name := []byte(file.Name)
svl, _ := t.Get(gk, nil) // skip error, we check len(svl) != 0 later
var fl VersionList
var oldFile protocol.FileInfo
var hasOldFile bool
// Remove the device from the current version list
if len(svl) != 0 {
fl.Unmarshal(svl) // skip error, range handles success case
for i := range fl.Versions {
if bytes.Equal(fl.Versions[i].Device, device) {
if fl.Versions[i].Version.Equal(file.Version) && fl.Versions[i].Invalid == file.Invalid {
// No need to do anything
return false
}
if i == 0 {
// Keep the current newest file around so we can subtract it from
// the metadata if we replace it.
oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
}
fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
break
}
}
if svl, err := t.Get(gk, nil); err == nil {
fl.Unmarshal(svl) // Ignore error, continue with empty fl
}
nv := FileVersion{
Device: device,
Version: file.Version,
Invalid: file.Invalid,
}
insertedAt := -1
// Find a position in the list to insert this file. The file at the front
// of the list is the newer, the "global".
insert:
for i := range fl.Versions {
switch fl.Versions[i].Version.Compare(file.Version) {
case protocol.Equal:
if nv.Invalid {
continue insert
}
fallthrough
case protocol.Lesser:
// The version at this point in the list is equal to or lesser
// ("older") than us. We insert ourselves in front of it.
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
break insert
case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
// The version at this point is in conflict with us. We must pull
// the actual file metadata to determine who wins. If we win, we
// insert ourselves in front of the loser here. (The "Lesser" and
// "Greater" in the condition above is just based on the device
// IDs in the version vector, which is not the only thing we use
// to determine the winner.)
//
// A surprise missing file entry here is counted as a win for us.
if of, ok := t.getFile(folder, fl.Versions[i].Device, name); !ok || file.WinsConflict(of) {
fl.Versions = insertVersion(fl.Versions, i, nv)
insertedAt = i
break insert
}
}
}
fl, removedFV, removedAt, insertedAt := fl.update(folder, device, file, t.db)
if insertedAt == -1 {
// We didn't find a position for an insert above, so append to the end.
fl.Versions = append(fl.Versions, nv)
insertedAt = len(fl.Versions) - 1
}
// Fixup the global size calculation.
if hasOldFile {
// We removed the previous newest version
meta.removeFile(globalDeviceID, oldFile)
if insertedAt == 0 {
// inserted a new newest version
meta.addFile(globalDeviceID, file)
} else {
// The previous second version is now the first
if newGlobal, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
meta.addFile(globalDeviceID, newGlobal)
}
}
} else if insertedAt == 0 {
// We just inserted a new newest version.
meta.addFile(globalDeviceID, file)
if len(fl.Versions) > 1 {
// The previous newest version is now at index 1, grab it from there.
if oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
meta.removeFile(globalDeviceID, oldFile)
}
}
l.Debugln("update global; same version, global unchanged")
return false
}
l.Debugf("new global after update: %v", fl)
if removedAt != 0 && insertedAt != 0 {
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
t.Put(gk, mustMarshal(&fl))
return true
}
name := []byte(file.Name)
// Remove the old global from the global size counter
var oldGlobalFV FileVersion
if removedAt == 0 {
oldGlobalFV = removedFV
} else if len(fl.Versions) > 1 {
// The previous newest version is now at index 1
oldGlobalFV = fl.Versions[1]
}
if oldFile, ok := t.getFile(folder, oldGlobalFV.Device, name); ok {
// A failure to get the file here is surprising and our
// global size data will be incorrect until a restart...
meta.removeFile(globalDeviceID, oldFile)
}
// Add the new global to the global size counter
var newGlobal protocol.FileInfo
if insertedAt == 0 {
// Inserted a new newest version
newGlobal = file
} else if new, ok := t.getFile(folder, fl.Versions[0].Device, name); ok {
// The previous second version is now the first
newGlobal = new
} else {
panic("This file must exist in the db")
}
meta.addFile(globalDeviceID, newGlobal)
// Fixup the list of files we need.
nk := t.db.needKey(folder, name)
hasNeeded, _ := t.db.Has(nk, nil)
if localFV, haveLocalFV := fl.Get(protocol.LocalDeviceID[:]); need(newGlobal, haveLocalFV, localFV.Version) {
if !hasNeeded {
l.Debugf("local need insert; folder=%q, name=%q", folder, name)
t.Put(nk, nil)
}
} else if hasNeeded {
l.Debugf("local need delete; folder=%q, name=%q", folder, name)
t.Delete(nk)
}
l.Debugf(`new global for "%v" after update: %v`, file.Name, fl)
t.Put(gk, mustMarshal(&fl))
return true
}
func need(global FileIntf, haveLocal bool, localVersion protocol.Vector) bool {
// We never need an invalid file.
if global.IsInvalid() {
return false
}
// We don't need a deleted file if we don't have it.
if global.IsDeleted() && !haveLocal {
return false
}
// We don't need the global file if we already have the same version.
if haveLocal && localVersion.Equal(global.FileVersion()) {
return false
}
return true
}
// removeFromGlobal removes the device from the global version list for the
// given file. If the version list is empty after this, the file entry is
// removed entirely.
@ -246,11 +219,13 @@ func (t readWriteTransaction) removeFromGlobal(gk, folder, device, file []byte,
}
}
func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
t := append(vl, FileVersion{})
copy(t[i+1:], t[i:])
t[i] = v
return t
func (t readWriteTransaction) deleteKeyPrefix(prefix []byte) {
dbi := t.NewIterator(util.BytesPrefix(prefix), nil)
for dbi.Next() {
t.Delete(dbi.Key())
t.checkFlush()
}
dbi.Release()
}
type marshaller interface {

1
lib/db/set.go
View File

@ -44,6 +44,7 @@ type FileIntf interface {
HasPermissionBits() bool
SequenceNo() int64
BlockSize() int
FileVersion() protocol.Vector
}
// The Iterator is called with either a protocol.FileInfo or a

4
lib/db/structs.go
View File

@ -75,6 +75,10 @@ func (f FileInfoTruncated) SequenceNo() int64 {
return f.Sequence
}
func (f FileInfoTruncated) FileVersion() protocol.Vector {
return f.Version
}
func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo {
return protocol.FileInfo{
Name: f.Name,

22
lib/db/testdata/v0.14.45-update0to3.db.jsons vendored Normal file
View File

@ -0,0 +1,22 @@
{"k":"AAAAAAAAAAABL25vdGdvb2Q=","v":"Cggvbm90Z29vZEoHCgUIARDoB1ACggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHw=="}
{"k":"AAAAAAAAAAABYQ==","v":"CgFhSgcKBQgBEOgHUAGCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4f"}
{"k":"AAAAAAAAAAACYg==","v":"CgFiSgcKBQgBEOkHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIA=="}
{"k":"AAAAAAAAAAACYw==","v":"CgFjOAFKBwoFCAEQ6geCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AAAAAAAAAAACZA==","v":"CgFkSgcKBQgBEOsHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
{"k":"AAAAAAAAAAADYw==","v":"CgFjSgcKBQgBEOoHggEiGiAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH4IBJBABGiABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fIIIBJBACGiACAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gIYIBJBADGiADBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIoIBJBAEGiAEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiI4IBJBAFGiAFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJIIBJBAGGiAGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJQ=="}
{"k":"AAAAAAAAAAADZA==","v":"CgFkOAFKBwoFCAEQ6weCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AAAAAAAAAAADaW52YWxpZA==","v":"CgdpbnZhbGlkOAFKBwoFCAEQ7AeCASIaIAABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fggEkEAEaIAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gggEkEAIaIAIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhggEkEAMaIAMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiggEkEAQaIAQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIj"}
{"k":"AQAAAAAvbm90Z29vZA==","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
{"k":"AQAAAABh","v":"CisKBwoFCAEQ6AcSIP//////////////////////////////////////////"}
{"k":"AQAAAABi","v":"CisKBwoFCAEQ6QcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/At"}
{"k":"AQAAAABj","v":"CisKBwoFCAEQ6gcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcCi0KBwoFCAEQ6gcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtGAE="}
{"k":"AQAAAABk","v":"CisKBwoFCAEQ6wcSIAIj5b8/Vx850vCTKUE+HcWcQZUIgmhv//rEL3j3A/AtCi0KBwoFCAEQ6wcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
{"k":"AQAAAABpbnZhbGlk","v":"Ci0KBwoFCAEQ7AcSIEeUA//e9Ja19eW8nAoVIh5wBzFkUJ+jB2GvYwlPb5RcGAE="}
{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHy9ub3Rnb29k","v":"AAAAAA=="}
{"k":"AgAAAAAAAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eH2E=","v":"AAAAAA=="}
{"k":"BgAAAAAAAAAA","v":"VXBkYXRlU2NoZW1hMHRvMw=="}
{"k":"BwAAAAAAAAAA","v":""}
{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
{"k":"BwAAAAIAAAAA","v":"AiPlvz9XHznS8JMpQT4dxZxBlQiCaG//+sQvePcD8C0="}
{"k":"BwAAAAMAAAAA","v":"R5QD/970lrX15bycChUiHnAHMWRQn6MHYa9jCU9vlFw="}
{"k":"CQAAAAA=","v":"CicIAjACigEg//////////////////////////////////////////8KJwgFMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAKKASACI+W/P1cfOdLwkylBPh3FnEGVCIJob//6xC949wPwLQolCAGKASBHlAP/3vSWtfXlvJwKFSIecAcxZFCfowdhr2MJT2+UXBCyn6iaw4HGlxU="}

82
lib/db/util_test.go
View File

@ -11,8 +11,6 @@ import (
"io"
"os"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/util"
@ -58,37 +56,51 @@ func openJSONS(file string) (*leveldb.DB, error) {
return db, nil
}
func generateIgnoredFilesDB() {
// This generates a database with files with invalid flags, local and
// remote, in the format used in 0.14.48.
// The following commented tests were used to generate jsons files to stdout for
// future tests and are kept here for reference (reuse).
db := OpenMemory()
fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
{ // invalid (ignored) file
Name: "foo",
Type: protocol.FileInfoTypeFile,
Invalid: true,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
},
{ // regular file
Name: "bar",
Type: protocol.FileInfoTypeFile,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
},
})
fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
{ // invalid file
Name: "baz",
Type: protocol.FileInfoTypeFile,
Invalid: true,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
},
{ // regular file
Name: "quux",
Type: protocol.FileInfoTypeFile,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
},
})
writeJSONS(os.Stdout, db.DB)
}
// TestGenerateIgnoredFilesDB generates a database with files with invalid flags,
// local and remote, in the format used in 0.14.48.
// func TestGenerateIgnoredFilesDB(t *testing.T) {
// db := OpenMemory()
// fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
// { // invalid (ignored) file
// Name: "foo",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
// },
// { // regular file
// Name: "bar",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
// },
// })
// fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
// { // invalid file
// Name: "baz",
// Type: protocol.FileInfoTypeFile,
// Invalid: true,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
// },
// { // regular file
// Name: "quux",
// Type: protocol.FileInfoTypeFile,
// Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
// },
// })
// writeJSONS(os.Stdout, db.DB)
// }
// TestGenerateUpdate0to3DB generates a database with files with invalid flags, prefixed
// by a slash and other files to test database migration from version 0 to 3, in the
// format used in 0.14.45.
// func TestGenerateUpdate0to3DB(t *testing.T) {
// db := OpenMemory()
// fs := NewFileSet(update0to3Folder, fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
// for devID, files := range haveUpdate0to3 {
// fs.Update(devID, files)
// }
// writeJSONS(os.Stdout, db.DB)
// }

2
lib/model/requests_test.go
View File

@ -346,7 +346,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) {
expected := map[string]struct{}{invIgn: {}, ign: {}, ignExisting: {}}
for _, f := range fs {
if _, ok := expected[f.Name]; !ok {
t.Fatalf("Unexpected file %v was added to index", f.Name)
t.Errorf("Unexpected file %v was added to index", f.Name)
}
if !f.Invalid {
t.Errorf("File %v wasn't marked as invalid", f.Name)

4
lib/protocol/bep_extensions.go
View File

@ -95,6 +95,10 @@ func (f FileInfo) SequenceNo() int64 {
return f.Sequence
}
func (f FileInfo) FileVersion() Vector {
return f.Version
}
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {